swift-1.13.1/0000775000175400017540000000000012323703665014076 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/0000775000175400017540000000000012323703665015055 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/probe/0000775000175400017540000000000012323703665016164 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/probe/test_object_handoff.py0000775000175400017540000002051112323703611022521 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from unittest import main, TestCase from uuid import uuid4 from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException from swift.common.manager import Manager from test.probe.common import kill_server, kill_servers, reset_environment, \ start_server class TestObjectHandoff(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def test_main(self): # Create container # Kill one container/obj primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication, ensuring we run the handoff node last so it # should remove its extra handoff partition # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) # Delete container/obj # Assert we can't head container/obj # Assert container/obj is not in the container listing, both indirectly # and directly # Restart the first container/obj primary server again # Assert it still has container/obj # Run object replication, ensuring we run the handoff node last so it # should remove its extra handoff partition # Assert primary node no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) # We've indirectly verified the handoff node has the object, but let's # directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) start_server(onode['port'], self.port2server, self.pids) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) # Run the extra server last so it'll remove its extra partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) kill_server(onode['port'], self.port2server, self.pids) client.delete_object(self.url, self.token, container, obj) exc = None try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) start_server(onode['port'], self.port2server, self.pids) direct_client.direct_get_object(onode, opart, self.account, container, obj) # Run the extra server last so it'll remove its extra partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) if __name__ == '__main__': main() swift-1.13.1/test/probe/test_account_get_fake_responses_match.py0000775000175400017540000000762512323703611026337 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import httplib import re import unittest from swiftclient import get_auth from test.probe.common import kill_servers, reset_environment from urlparse import urlparse class TestAccountGetFakeResponsesMatch(unittest.TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() self.url, self.token = get_auth( 'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin') def tearDown(self): kill_servers(self.port2server, self.pids) def _account_path(self, account): _, _, path, _, _, _ = urlparse(self.url) basepath, _ = path.rsplit('/', 1) return basepath + '/' + account def _get(self, *a, **kw): kw['method'] = 'GET' return self._account_request(*a, **kw) def _account_request(self, account, method, headers=None): if headers is None: headers = {} headers['X-Auth-Token'] = self.token scheme, netloc, path, _, _, _ = urlparse(self.url) host, port = netloc.split(':') port = int(port) conn = httplib.HTTPConnection(host, port) conn.request(method, self._account_path(account), headers=headers) resp = conn.getresponse() if resp.status // 100 != 2: raise Exception("Unexpected status %s\n%s" % (resp.status, resp.read())) response_headers = dict(resp.getheaders()) response_body = resp.read() resp.close() return response_headers, response_body def test_main(self): # Two accounts: "real" and "fake". The fake one doesn't have any .db # files on disk; the real one does. The real one is empty. # # Make sure the important response fields match. real_acct = "AUTH_real" fake_acct = "AUTH_fake" self._account_request(real_acct, 'POST', {'X-Account-Meta-Bert': 'Ernie'}) # text real_headers, real_body = self._get(real_acct) fake_headers, fake_body = self._get(fake_acct) self.assertEqual(real_body, fake_body) self.assertEqual(real_headers['content-type'], fake_headers['content-type']) # json real_headers, real_body = self._get( real_acct, headers={'Accept': 'application/json'}) fake_headers, fake_body = self._get( fake_acct, headers={'Accept': 'application/json'}) self.assertEqual(real_body, fake_body) self.assertEqual(real_headers['content-type'], fake_headers['content-type']) # xml real_headers, real_body = self._get( real_acct, headers={'Accept': 'application/xml'}) fake_headers, fake_body = self._get( fake_acct, headers={'Accept': 'application/xml'}) # the account name is in the XML response real_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', real_body) fake_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', fake_body) self.assertEqual(real_body, fake_body) self.assertEqual(real_headers['content-type'], fake_headers['content-type']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/probe/test_container_failures.py0000775000175400017540000001620712323703611023451 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from os import listdir from os.path import join as path_join from unittest import main, TestCase from uuid import uuid4 from eventlet import GreenPool, Timeout import eventlet from sqlite3 import connect from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException from swift.common.utils import hash_path, readconf from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server eventlet.monkey_patch(all=False, socket=True) def get_db_file_path(obj_dir): files = sorted(listdir(obj_dir), reverse=True) for filename in files: if filename.endswith('db'): return path_join(obj_dir, filename) class TestContainerFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals( [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1]], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3') def test_two_nodes_fail(self): # Create container1 # Kill container1 servers excepting one of the primaries # Delete container1 directly to the one primary still up # Restart other container1 servers # Get to a final state # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) # Assert account level also indicates container1 is gone container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) kill_server(cnodes[1]['port'], self.port2server, self.pids) direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) start_server(cnodes[1]['port'], self.port2server, self.pids) start_server(cnp_port, self.port2server, self.pids) get_to_final_state() for cnode in cnodes: exc = None try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '0') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') def _get_container_db_files(self, container): opart, onodes = self.container_ring.get_nodes(self.account, container) onode = onodes[0] db_files = [] for onode in onodes: node_id = (onode['port'] - 6000) / 10 device = onode['device'] hash_str = hash_path(self.account, container) server_conf = readconf(self.configs['container-server'][node_id]) devices = server_conf['app:container-server']['devices'] obj_dir = '%s/%s/containers/%s/%s/%s/' % (devices, device, opart, hash_str[-3:], hash_str) db_files.append(get_db_file_path(obj_dir)) return db_files def test_locked_container_dbs(self): def run_test(num_locks, catch_503): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) db_files = self._get_container_db_files(container) db_conns = [] for i in range(num_locks): db_conn = connect(db_files[i]) db_conn.execute('begin exclusive transaction') db_conns.append(db_conn) if catch_503: exc = None try: client.delete_container(self.url, self.token, container) except client.ClientException as err: exc = err self.assertEquals(exc.http_status, 503) else: client.delete_container(self.url, self.token, container) pool = GreenPool() try: with Timeout(15): pool.spawn(run_test, 1, False) pool.spawn(run_test, 2, True) pool.spawn(run_test, 3, True) pool.waitall() except Timeout as err: raise Exception( "The server did not return a 503 on container db locks, " "it just hangs: %s" % err) if __name__ == '__main__': main() swift-1.13.1/test/probe/test_account_failures.py0000775000175400017540000001771712323703611023132 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from unittest import main, TestCase from swiftclient import client from swift.common import direct_client from swift.common.manager import Manager from test.probe.common import get_to_final_state, kill_nonprimary_server, \ kill_server, kill_servers, reset_environment, start_server class TestAccountFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def test_main(self): # Create container1 and container2 # Assert account level sees them # Create container2/object1 # Assert account level doesn't see it yet # Get to final state # Assert account level now sees the container2/object1 # Kill account servers excepting two of the primaries # Delete container1 # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet # Put container2/object2 # Run container updaters # Assert account level now knows about container2/object2 # Restart other primary account server # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet # Get to final state # Assert that server is now up to date container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.port2server, self.pids) kill_server(anodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(not found1) self.assert_(found2) Manager(['container-updater']).once() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2) start_server(anodes[0]['port'], self.port2server, self.pids) headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2) if __name__ == '__main__': main() swift-1.13.1/test/probe/common.py0000664000175400017540000002173012323703611020020 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from httplib import HTTPConnection import os from subprocess import Popen, PIPE import sys from time import sleep, time from collections import defaultdict from swiftclient import get_auth, head_account from swift.common.ring import Ring from swift.common.utils import readconf from swift.common.manager import Manager from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC def get_server_number(port, port2server): server_number = port2server[port] server, number = server_number[:-1], server_number[-1:] try: number = int(number) except ValueError: # probably the proxy return server_number, None return server, number def start_server(port, port2server, pids, check=True): server, number = get_server_number(port, port2server) err = Manager([server]).start(number=number, wait=False) if err: raise Exception('unable to start %s' % ( server if not number else '%s%s' % (server, number))) if check: return check_server(port, port2server, pids) return None def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT): server = port2server[port] if server[:-1] in ('account', 'container', 'object'): if int(server[-1]) > 4: return None path = '/connect/1/2' if server[:-1] == 'container': path += '/3' elif server[:-1] == 'object': path += '/3/4' try_until = time() + timeout while True: try: conn = HTTPConnection('127.0.0.1', port) conn.request('GET', path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception( 'Unexpected status %s' % resp.status) break except Exception as err: if time() > try_until: print err print 'Giving up on %s:%s after %s seconds.' % ( server, port, timeout) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print err print 'Giving up on proxy:8080 after 30 seconds.' raise err sleep(0.1) return None def kill_server(port, port2server, pids): server, number = get_server_number(port, port2server) err = Manager([server]).kill(number=number) if err: raise Exception('unable to kill %s' % (server if not number else '%s%s' % (server, number))) try_until = time() + 30 while True: try: conn = HTTPConnection('127.0.0.1', port) conn.request('GET', '/') conn.getresponse() except Exception as err: break if time() > try_until: raise Exception( 'Still answering on port %s after 30 seconds' % port) sleep(0.1) def kill_servers(port2server, pids): Manager(['all']).kill() def kill_nonprimary_server(primary_nodes, port2server, pids): primary_ports = [n['port'] for n in primary_nodes] for port, server in port2server.iteritems(): if port in primary_ports: server_type = server[:-1] break else: raise Exception('Cannot figure out server type for %r' % primary_nodes) for port, server in list(port2server.iteritems()): if server[:-1] == server_type and port not in primary_ports: kill_server(port, port2server, pids) return port def get_ring(server, force_validate=None): ring = Ring('/etc/swift/%s.ring.gz' % server) if not VALIDATE_RSYNC and not force_validate: return ring # easy sanity checks assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % ( ring.serialized_path, ring.replica_count) assert 4 == len(ring.devs), '%s has %s devices instead of 4' % ( ring.serialized_path, len(ring.devs)) # map server to config by port port_to_config = {} for server_ in Manager([server]): for config_path in server_.conf_files(): conf = readconf(config_path, section_name='%s-replicator' % server_.type) port_to_config[int(conf['bind_port'])] = conf for dev in ring.devs: # verify server is exposing mounted device conf = port_to_config[dev['port']] for device in os.listdir(conf['devices']): if device == dev['device']: dev_path = os.path.join(conf['devices'], device) full_path = os.path.realpath(dev_path) assert os.path.exists(full_path), \ 'device %s in %s was not found (%s)' % ( device, conf['devices'], full_path) break else: raise AssertionError( "unable to find ring device %s under %s's devices (%s)" % ( dev['device'], server, conf['devices'])) # verify server is exposing rsync device if port_to_config[dev['port']].get('vm_test_mode', False): rsync_export = '%s%s' % (server, dev['replication_port']) else: rsync_export = server cmd = "rsync rsync://localhost/%s" % rsync_export p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: raise AssertionError('unable to connect to rsync ' 'export %s (%s)' % (rsync_export, cmd)) for line in stdout.splitlines(): if line.rsplit(None, 1)[-1] == dev['device']: break else: raise AssertionError("unable to find ring device %s under rsync's " "exported devices for %s (%s)" % ( dev['device'], rsync_export, cmd)) return ring def reset_environment(): p = Popen("resetswift 2>&1", shell=True, stdout=PIPE) stdout, _stderr = p.communicate() print stdout Manager(['all']).stop() pids = {} try: account_ring = get_ring('account') container_ring = get_ring('container') object_ring = get_ring('object') Manager(['main']).start(wait=False) port2server = {} for server, port in [('account', 6002), ('container', 6001), ('object', 6000)]: for number in xrange(1, 9): port2server[port + (number * 10)] = '%s%d' % (server, number) for port in port2server: check_server(port, port2server, pids) port2server[8080] = 'proxy' url, token, account = check_server(8080, port2server, pids) config_dict = defaultdict(dict) for name in ('account', 'container', 'object'): for server_name in (name, '%s-replicator' % name): for server in Manager([server_name]): for i, conf in enumerate(server.conf_files(), 1): config_dict[server.server][i] = conf except BaseException: try: raise except AssertionError as e: print >>sys.stderr, 'ERROR: %s' % e os._exit(1) finally: try: kill_servers(port2server, pids) except Exception: pass return pids, port2server, account_ring, container_ring, object_ring, url, \ token, account, config_dict def get_to_final_state(): replicators = Manager(['account-replicator', 'container-replicator', 'object-replicator']) replicators.stop() updaters = Manager(['container-updater', 'object-updater']) updaters.stop() replicators.once() updaters.once() replicators.once() if __name__ == "__main__": for server in ('account', 'container', 'object'): get_ring(server, force_validate=True) print '%s OK' % server swift-1.13.1/test/probe/test_empty_device_handoff.py0000775000175400017540000001604412323703611023736 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import time from unittest import main, TestCase from uuid import uuid4 from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException from test.probe.common import kill_server, kill_servers, reset_environment,\ start_server from swift.common.utils import readconf from swift.common.manager import Manager class TestEmptyDevice(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def _get_objects_dir(self, onode): device = onode['device'] node_id = (onode['port'] - 6000) / 10 obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] obj_dir = '%s/%s' % (devices, device) return obj_dir def test_main(self): # Create container # Kill one container/obj primary server # Delete the "objects" directory on the primary server # Create container/obj (goes to two primary servers and one handoff) # Kill other two container/obj primary servers # Indirectly through proxy assert we can get container/obj # Restart those other two container/obj primary servers # Directly to handoff server assert we can get container/obj # Assert container listing (via proxy and directly) has container/obj # Bring the first container/obj primary server back up # Assert that it doesn't have container/obj yet # Run object replication for first container/obj primary server # Run object replication for handoff node # Assert the first container/obj primary server now has container/obj # Assert the handoff server no longer has container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) obj_dir = '%s/objects' % self._get_objects_dir(onode) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') timeout = time.time() + 5 found_objs_on_cnode = [] while time.time() < timeout: for cnode in [c for c in cnodes if cnodes not in found_objs_on_cnode]: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: found_objs_on_cnode.append(cnode) if len(found_objs_on_cnode) >= len(cnodes): break time.sleep(0.3) if len(found_objs_on_cnode) < len(cnodes): missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes if cnode not in found_objs_on_cnode] raise Exception('Container servers %r did not know about object' % missing) start_server(onode['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) exc = None try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) try: port_num = onode['replication_port'] except KeyError: port_num = onode['port'] try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] num = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=num) another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except ClientException as err: exc = err self.assertEquals(exc.http_status, 404) if __name__ == '__main__': main() swift-1.13.1/test/probe/test_replication_servers_working.py0000664000175400017540000001674112323703611025417 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from unittest import main, TestCase from uuid import uuid4 import os import time import shutil from swiftclient import client from test.probe.common import kill_servers, reset_environment from swift.common.utils import readconf from swift.common.manager import Manager def collect_info(path_list): """ Recursive collect dirs and files in path_list directory. :param path_list: start directory for collecting :return files_list, dir_list: tuple of included directories and files """ files_list = [] dir_list = [] for path in path_list: temp_files_list = [] temp_dir_list = [] for root, dirs, files in os.walk(path): temp_files_list += files temp_dir_list += dirs files_list.append(temp_files_list) dir_list.append(temp_dir_list) return files_list, dir_list def find_max_occupancy_node(dir_list): """ Find node with maximum occupancy. :param list_dir: list of directories for each node. :return number: number node in list_dir """ count = 0 number = 0 length = 0 for dirs in dir_list: if length < len(dirs): length = len(dirs) number = count count += 1 return number class TestReplicatorFunctions(TestCase): """ Class for testing replicators and replication servers. By default configuration - replication servers not used. For testing separete replication servers servers need to change ring's files using set_info command or new ring's files with different port values. """ def setUp(self): """ Reset all environment and start all servers. """ (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): """ Stop all servers. """ kill_servers(self.port2server, self.pids) def test_main(self): # Create one account, container and object file. # Find node with account, container and object replicas. # Delete all directories and files from this node (device). # Wait 60 seconds and check replication results. # Delete directories and files in objects storage without # deleting file "hashes.pkl". # Check, that files not replicated. # Delete file "hashes.pkl". # Check, that all files were replicated. path_list = [] # Figure out where the devices are for node_id in range(1, 5): conf = readconf(self.configs['object-server'][node_id]) device_path = conf['app:object-server']['devices'] for dev in self.object_ring.devs: if dev['port'] == int(conf['app:object-server']['bind_port']): device = dev['device'] path_list.append(os.path.join(device_path, device)) # Put data to storage nodes container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, 'VERIFY') # Get all data file information (files_list, dir_list) = collect_info(path_list) num = find_max_occupancy_node(dir_list) test_node = path_list[num] test_node_files_list = [] for files in files_list[num]: if not files.endswith('.pending'): test_node_files_list.append(files) test_node_dir_list = dir_list[num] # Run all replicators try: Manager(['object-replicator', 'container-replicator', 'account-replicator']).start() # Delete some files for directory in os.listdir(test_node): shutil.rmtree(os.path.join(test_node, directory)) self.assertFalse(os.listdir(test_node)) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: (new_files_list, new_dir_list) = collect_info([test_node]) try: # Check replicate files and dir for files in test_node_files_list: self.assertTrue(files in new_files_list[0]) for dir in test_node_dir_list: self.assertTrue(dir in new_dir_list[0]) break except Exception: if time.time() - begin > 60: raise time.sleep(1) # Check behavior by deleting hashes.pkl file for directory in os.listdir(os.path.join(test_node, 'objects')): for input_dir in os.listdir(os.path.join( test_node, 'objects', directory)): if os.path.isdir(os.path.join( test_node, 'objects', directory, input_dir)): shutil.rmtree(os.path.join( test_node, 'objects', directory, input_dir)) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: try: for directory in os.listdir(os.path.join( test_node, 'objects')): for input_dir in os.listdir(os.path.join( test_node, 'objects', directory)): self.assertFalse(os.path.isdir( os.path.join(test_node, 'objects', directory, '/', input_dir))) break except Exception: if time.time() - begin > 60: raise time.sleep(1) for directory in os.listdir(os.path.join(test_node, 'objects')): os.remove(os.path.join( test_node, 'objects', directory, 'hashes.pkl')) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: try: (new_files_list, new_dir_list) = collect_info([test_node]) # Check replicate files and dirs for files in test_node_files_list: self.assertTrue(files in new_files_list[0]) for directory in test_node_dir_list: self.assertTrue(directory in new_dir_list[0]) break except Exception: if time.time() - begin > 60: raise time.sleep(1) finally: Manager(['object-replicator', 'container-replicator', 'account-replicator']).stop() if __name__ == '__main__': main() swift-1.13.1/test/probe/test_object_async_update.py0000775000175400017540000000462212323703611023600 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from unittest import main, TestCase from uuid import uuid4 from swiftclient import client from swift.common import direct_client from swift.common.manager import Manager from test.probe.common import kill_nonprimary_server, kill_server, \ kill_servers, reset_environment, start_server class TestObjectAsyncUpdate(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def test_main(self): # Create container # Kill container servers excepting two of the primaries # Create container/obj # Restart other primary server # Assert it does not know about container/obj # Run the object-updaters # Assert the other primary server now knows about container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnode['port'], self.port2server, self.pids) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') start_server(cnode['port'], self.port2server, self.pids) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) Manager(['object-updater']).once() objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs) if __name__ == '__main__': main() swift-1.13.1/test/probe/test_object_failures.py0000775000175400017540000001633412323703611022736 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import time from os import listdir, unlink from os.path import join as path_join from unittest import main, TestCase from uuid import uuid4 from swiftclient import client from swift.common import direct_client from swift.common.exceptions import ClientException from swift.common.utils import hash_path, readconf from swift.obj.diskfile import write_metadata, read_metadata from test.probe.common import kill_servers, reset_environment RETRIES = 5 def get_data_file_path(obj_dir): files = [] # We might need to try a few times if a request hasn't yet settled. For # instance, a PUT can return success when just 2 of 3 nodes has completed. for attempt in xrange(RETRIES + 1): try: files = sorted(listdir(obj_dir), reverse=True) break except Exception: if attempt < RETRIES: time.sleep(1) else: raise for filename in files: return path_join(obj_dir, filename) class TestObjectFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account, self.configs) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def _setup_data_file(self, container, obj, data): client.put_container(self.url, self.token, container) client.put_object(self.url, self.token, container, obj, data) odata = client.get_object(self.url, self.token, container, obj)[-1] self.assertEquals(odata, data) opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] node_id = (onode['port'] - 6000) / 10 device = onode['device'] hash_str = hash_path(self.account, container, obj) obj_server_conf = readconf(self.configs['object-server'][node_id]) devices = obj_server_conf['app:object-server']['devices'] obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices, device, opart, hash_str[-3:], hash_str) data_file = get_data_file_path(obj_dir) return onode, opart, data_file def run_quarantine(self): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'VERIFY') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) odata = direct_client.direct_get_object( onode, opart, self.account, container, obj)[-1] self.assertEquals(odata, 'VERIFY') try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_range_etag(self): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'RANGE') metadata = read_metadata(data_file) metadata['ETag'] = 'badetag' write_metadata(data_file, metadata) for header, result in [({'Range': 'bytes=0-2'}, 'RAN'), ({'Range': 'bytes=1-11'}, 'ANGE'), ({'Range': 'bytes=0-11'}, 'RANGE')]: odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers=header)[-1] self.assertEquals(odata, result) try: direct_client.direct_get_object(onode, opart, self.account, container, obj) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_get(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_get_object(onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_head(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_head_object(onode, opart, self.account, container, obj, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) def run_quarantine_zero_byte_post(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') metadata = read_metadata(data_file) unlink(data_file) with open(data_file, 'w') as fpointer: write_metadata(fpointer, metadata) try: direct_client.direct_post_object( onode, opart, self.account, container, obj, {'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}, conn_timeout=1, response_timeout=1) raise Exception("Did not quarantine object") except ClientException as err: self.assertEquals(err.http_status, 404) def test_runner(self): self.run_quarantine() self.run_quarantine_range_etag() self.run_quarantine_zero_byte_get() self.run_quarantine_zero_byte_head() self.run_quarantine_zero_byte_post() if __name__ == '__main__': main() swift-1.13.1/test/probe/__init__.py0000664000175400017540000000037212323703611020266 0ustar jenkinsjenkins00000000000000from test import get_config from swift.common.utils import config_true_value config = get_config('probe_test') CHECK_SERVER_TIMEOUT = int(config.get('check_server_timeout', 30)) VALIDATE_RSYNC = config_true_value(config.get('validate_rsync', False)) swift-1.13.1/test/unit/0000775000175400017540000000000012323703665016034 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/account/0000775000175400017540000000000012323703665017470 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/account/test_server.py0000664000175400017540000024331312323703614022407 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import errno import os import mock import unittest from tempfile import mkdtemp from shutil import rmtree from StringIO import StringIO from test.unit import FakeLogger import simplejson import xml.dom.minidom from swift.common.swob import Request from swift.account.server import AccountController, ACCOUNT_LISTING_LIMIT from swift.common.utils import normalize_timestamp, replication, public from swift.common.request_helpers import get_sys_meta_prefix class TestAccountController(unittest.TestCase): """Test swift.account.server.AccountController""" def setUp(self): """Set up for testing swift.account.server.AccountController""" self.testdir_base = mkdtemp() self.testdir = os.path.join(self.testdir_base, 'account_server') self.controller = AccountController( {'devices': self.testdir, 'mount_check': 'false'}) def tearDown(self): """Tear down for testing swift.account.server.AccountController""" try: rmtree(self.testdir_base) except OSError as err: if err.errno != errno.ENOENT: raise def test_DELETE_not_found(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue('X-Account-Status' not in resp.headers) def test_DELETE_empty(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_DELETE_not_empty(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) # We now allow deleting non-empty accounts self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_DELETE_now_empty(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank( '/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '2', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_DELETE_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_DELETE_timestamp_not_float(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': 'not-float'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_DELETE_insufficient_storage(self): self.controller = AccountController({'devices': self.testdir}) req = Request.blank( '/sda-null/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 507) def test_HEAD_not_found(self): # Test the case in which account does not exist (can be recreated) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue('X-Account-Status' not in resp.headers) # Test the case in which account was deleted but not yet reaped req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_HEAD_empty_account(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['x-account-container-count'], '0') self.assertEqual(resp.headers['x-account-object-count'], '0') self.assertEqual(resp.headers['x-account-bytes-used'], '0') def test_HEAD_with_containers(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['x-account-container-count'], '2') self.assertEqual(resp.headers['x-account-object-count'], '0') self.assertEqual(resp.headers['x-account-bytes-used'], '0') req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '1', 'X-Bytes-Used': '2', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '5'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['x-account-container-count'], '2') self.assertEqual(resp.headers['x-account-object-count'], '4') self.assertEqual(resp.headers['x-account-bytes-used'], '6') def test_HEAD_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_HEAD_invalid_content_type(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}, headers={'Accept': 'application/plain'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 406) def test_HEAD_insufficient_storage(self): self.controller = AccountController({'devices': self.testdir}) req = Request.blank('/sda-null/p/a', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 507) def test_HEAD_invalid_format(self): format = '%D1%BD%8A9' # invalid UTF-8; should be %E1%BD%8A9 (E -> D) req = Request.blank('/sda1/p/a?format=' + format, environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_PUT_not_found(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-PUT-Timestamp': normalize_timestamp(1), 'X-DELETE-Timestamp': normalize_timestamp(0), 'X-Object-Count': '1', 'X-Bytes-Used': '1', 'X-Timestamp': normalize_timestamp(0)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue('X-Account-Status' not in resp.headers) def test_PUT(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) def test_PUT_simulated_create_race(self): state = ['initial'] from swift.account.backend import AccountBroker as OrigAcBr class InterceptedAcBr(OrigAcBr): def __init__(self, *args, **kwargs): super(InterceptedAcBr, self).__init__(*args, **kwargs) if state[0] == 'initial': # Do nothing initially pass elif state[0] == 'race': # Save the original db_file attribute value self._saved_db_file = self.db_file self.db_file += '.doesnotexist' def initialize(self, *args, **kwargs): if state[0] == 'initial': # Do nothing initially pass elif state[0] == 'race': # Restore the original db_file attribute to get the race # behavior self.db_file = self._saved_db_file return super(InterceptedAcBr, self).initialize(*args, **kwargs) with mock.patch("swift.account.server.AccountBroker", InterceptedAcBr): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) state[0] = "race" req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) def test_PUT_after_DELETE(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(2)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 403) self.assertEqual(resp.body, 'Recently deleted') self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_PUT_GET_metadata(self): # Set metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Account-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Account-Meta-Test2': 'Value2'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'Value') self.assertEqual(resp.headers.get('x-account-meta-test2'), 'Value2') # Update metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(3), 'X-Account-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(2), 'X-Account-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(4), 'X-Account-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) def test_PUT_GET_sys_metadata(self): prefix = get_sys_meta_prefix('account') hdr = '%stest' % prefix hdr2 = '%stest2' % prefix # Set metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1), hdr.title(): 'Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), hdr2.title(): 'Value2'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'Value') self.assertEqual(resp.headers.get(hdr2), 'Value2') # Update metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(3), hdr.title(): 'New Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(2), hdr.title(): 'Old Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(4), hdr.title(): ''}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assert_(hdr not in resp.headers) def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_PUT_insufficient_storage(self): self.controller = AccountController({'devices': self.testdir}) req = Request.blank('/sda-null/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 507) def test_POST_HEAD_metadata(self): req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) # Set metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Account-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'Value') # Update metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(3), 'X-Account-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(2), 'X-Account-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get('x-account-meta-test'), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(4), 'X-Account-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assert_('x-account-meta-test' not in resp.headers) def test_POST_HEAD_sys_metadata(self): prefix = get_sys_meta_prefix('account') hdr = '%stest' % prefix req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) # Set metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), hdr.title(): 'Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'Value') # Update metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(3), hdr.title(): 'New Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(2), hdr.title(): 'Old Value'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers.get(hdr), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(4), hdr.title(): ''}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assert_(hdr not in resp.headers) def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_POST_timestamp_not_float(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '0'}, headers={'X-Timestamp': 'not-float'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 400) def test_POST_insufficient_storage(self): self.controller = AccountController({'devices': self.testdir}) req = Request.blank('/sda-null/p/a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 507) def test_POST_after_DELETE_not_found(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '2'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_GET_not_found_plain(self): # Test the case in which account does not exist (can be recreated) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue('X-Account-Status' not in resp.headers) # Test the case in which account was deleted but not yet reaped req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertEqual(resp.headers['X-Account-Status'], 'Deleted') def test_GET_not_found_json(self): req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) def test_GET_not_found_xml(self): req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) def test_GET_empty_account_plain(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 204) self.assertEqual(resp.headers['Content-Type'], 'text/plain; charset=utf-8') def test_GET_empty_account_json(self): req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.headers['Content-Type'], 'application/json; charset=utf-8') def test_GET_empty_account_xml(self): req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.headers['Content-Type'], 'application/xml; charset=utf-8') def test_GET_over_limit(self): req = Request.blank( '/sda1/p/a?limit=%d' % (ACCOUNT_LISTING_LIMIT + 1), environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 412) def test_GET_with_containers_plain(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.body.strip().split('\n'), ['c1', 'c2']) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '1', 'X-Bytes-Used': '2', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.body.strip().split('\n'), ['c1', 'c2']) self.assertEqual(resp.content_type, 'text/plain') self.assertEqual(resp.charset, 'utf-8') # test unknown format uses default plain req = Request.blank('/sda1/p/a?format=somethinglese', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(resp.body.strip().split('\n'), ['c1', 'c2']) self.assertEqual(resp.content_type, 'text/plain') self.assertEqual(resp.charset, 'utf-8') def test_GET_with_containers_json(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(simplejson.loads(resp.body), [{'count': 0, 'bytes': 0, 'name': 'c1'}, {'count': 0, 'bytes': 0, 'name': 'c2'}]) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '1', 'X-Bytes-Used': '2', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) self.assertEqual(simplejson.loads(resp.body), [{'count': 1, 'bytes': 2, 'name': 'c1'}, {'count': 3, 'bytes': 4, 'name': 'c2'}]) self.assertEqual(resp.content_type, 'application/json') self.assertEqual(resp.charset, 'utf-8') def test_GET_with_containers_xml(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.content_type, 'application/xml') self.assertEqual(resp.status_int, 200) dom = xml.dom.minidom.parseString(resp.body) self.assertEqual(dom.firstChild.nodeName, 'account') listing = \ [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] self.assertEqual(len(listing), 2) self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), ['bytes', 'count', 'name']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c1') node = [n for n in container if n.nodeName == 'count'][0] self.assertEqual(node.firstChild.nodeValue, '0') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEqual(node.firstChild.nodeValue, '0') self.assertEqual(listing[-1].nodeName, 'container') container = \ [n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), ['bytes', 'count', 'name']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c2') node = [n for n in container if n.nodeName == 'count'][0] self.assertEqual(node.firstChild.nodeValue, '0') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEqual(node.firstChild.nodeValue, '0') req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '1', 'X-Bytes-Used': '2', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '2', 'X-Delete-Timestamp': '0', 'X-Object-Count': '3', 'X-Bytes-Used': '4', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a?format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 200) dom = xml.dom.minidom.parseString(resp.body) self.assertEqual(dom.firstChild.nodeName, 'account') listing = \ [n for n in dom.firstChild.childNodes if n.nodeName != '#text'] self.assertEqual(len(listing), 2) self.assertEqual(listing[0].nodeName, 'container') container = [n for n in listing[0].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), ['bytes', 'count', 'name']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c1') node = [n for n in container if n.nodeName == 'count'][0] self.assertEqual(node.firstChild.nodeValue, '1') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEqual(node.firstChild.nodeValue, '2') self.assertEqual(listing[-1].nodeName, 'container') container = [ n for n in listing[-1].childNodes if n.nodeName != '#text'] self.assertEqual(sorted([n.nodeName for n in container]), ['bytes', 'count', 'name']) node = [n for n in container if n.nodeName == 'name'][0] self.assertEqual(node.firstChild.nodeValue, 'c2') node = [n for n in container if n.nodeName == 'count'][0] self.assertEqual(node.firstChild.nodeValue, '3') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEqual(node.firstChild.nodeValue, '4') self.assertEqual(resp.charset, 'utf-8') def test_GET_xml_escapes_account_name(self): req = Request.blank( '/sda1/p/%22%27', # "' environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank( '/sda1/p/%22%27?format=xml', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) dom = xml.dom.minidom.parseString(resp.body) self.assertEqual(dom.firstChild.attributes['name'].value, '"\'') def test_GET_xml_escapes_container_name(self): req = Request.blank( '/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank( '/sda1/p/a/%22%3Cword', # "

Method Not Allowed

The method is not ' 'allowed for this resource.

'] mock_method = replication(public(lambda x: mock.MagicMock())) with mock.patch.object(self.controller, method, new=mock_method): mock_method.replication = True response = self.controller.__call__(env, start_response) self.assertEqual(response, answer) def test_GET_log_requests_true(self): self.controller.logger = FakeLogger() self.controller.log_requests = True req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue(self.controller.logger.log_dict['info']) def test_GET_log_requests_false(self): self.controller.logger = FakeLogger() self.controller.log_requests = False req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertFalse(self.controller.logger.log_dict['info']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/account/test_reaper.py0000664000175400017540000004317612323703611022361 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time import shutil import tempfile import unittest from logging import DEBUG from mock import patch from contextlib import nested from swift.account import reaper from swift.account.backend import DATADIR from swift.common.exceptions import ClientException from swift.common.utils import normalize_timestamp class FakeLogger(object): def __init__(self, *args, **kwargs): self.inc = {'return_codes.4': 0, 'return_codes.2': 0, 'objects_failures': 0, 'objects_deleted': 0, 'objects_remaining': 0, 'objects_possibly_remaining': 0, 'containers_failures': 0, 'containers_deleted': 0, 'containers_remaining': 0, 'containers_possibly_remaining': 0} self.exp = [] def info(self, msg, *args): self.msg = msg def timing_since(*args, **kwargs): pass def getEffectiveLevel(self): return DEBUG def exception(self, *args): self.exp.append(args) def increment(self, key): self.inc[key] += 1 class FakeBroker(object): def __init__(self): self.info = {} def get_info(self): return self.info class FakeAccountBroker(object): def __init__(self, containers): self.containers = containers def get_info(self): info = {'account': 'a', 'delete_timestamp': time.time() - 10} return info def list_containers_iter(self, *args): for cont in self.containers: yield cont, None, None, None def is_status_deleted(self): return True def empty(self): return False class FakeRing(object): def __init__(self): self.nodes = [{'id': '1', 'ip': '10.10.10.1', 'port': None, 'device': None}, {'id': '2', 'ip': '10.10.10.1', 'port': None, 'device': None}, {'id': '3', 'ip': '10.10.10.1', 'port': None, 'device': None}, ] def get_nodes(self, *args, **kwargs): return ('partition', self.nodes) def get_part_nodes(self, *args, **kwargs): return self.nodes acc_nodes = [{'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}] cont_nodes = [{'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}, {'device': 'sda1', 'ip': '', 'port': ''}] class TestReaper(unittest.TestCase): def setUp(self): self.to_delete = [] self.myexp = ClientException("", http_host=None, http_port=None, http_device=None, http_status=404, http_reason=None ) def tearDown(self): for todel in self.to_delete: shutil.rmtree(todel) def fake_direct_delete_object(self, *args, **kwargs): if self.amount_fail < self.max_fail: self.amount_fail += 1 raise self.myexp def fake_object_ring(self): return FakeRing() def fake_direct_delete_container(self, *args, **kwargs): if self.amount_delete_fail < self.max_delete_fail: self.amount_delete_fail += 1 raise self.myexp def fake_direct_get_container(self, *args, **kwargs): if self.get_fail: raise self.myexp objects = [{'name': 'o1'}, {'name': 'o2'}, {'name': unicode('o3')}, {'name': ''}] return None, objects def fake_container_ring(self): return FakeRing() def fake_reap_object(self, *args, **kwargs): if self.reap_obj_fail: raise Exception def prepare_data_dir(self, ts=False): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, 'sda1', DATADIR) os.makedirs(path) path = os.path.join(path, '100', 'a86', 'a8c682d2472e1720f2d81ff8993aba6') os.makedirs(path) suffix = 'db' if ts: suffix = 'ts' with open(os.path.join(path, 'a8c682203aba6.%s' % suffix), 'w') as fd: fd.write('') return devices_path def init_reaper(self, conf={}, myips=['10.10.10.1'], fakelogger=False): r = reaper.AccountReaper(conf) r.stats_return_codes = {} r.stats_containers_deleted = 0 r.stats_containers_remaining = 0 r.stats_containers_possibly_remaining = 0 r.stats_objects_deleted = 0 r.stats_objects_remaining = 0 r.stats_objects_possibly_remaining = 0 r.myips = myips if fakelogger: r.logger = FakeLogger() return r def fake_reap_account(self, *args, **kwargs): self.called_amount += 1 def fake_account_ring(self): return FakeRing() def test_delay_reaping_conf_default(self): r = reaper.AccountReaper({}) self.assertEqual(r.delay_reaping, 0) r = reaper.AccountReaper({'delay_reaping': ''}) self.assertEqual(r.delay_reaping, 0) def test_delay_reaping_conf_set(self): r = reaper.AccountReaper({'delay_reaping': '123'}) self.assertEqual(r.delay_reaping, 123) def test_delay_reaping_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'delay_reaping': 'abc'}) def test_reap_warn_after_conf_set(self): conf = {'delay_reaping': '2', 'reap_warn_after': '3'} r = reaper.AccountReaper(conf) self.assertEqual(r.reap_not_done_after, 5) def test_reap_warn_after_conf_bad_value(self): self.assertRaises(ValueError, reaper.AccountReaper, {'reap_warn_after': 'abc'}) def test_reap_delay(self): time_value = [100] def _time(): return time_value[0] time_orig = reaper.time try: reaper.time = _time r = reaper.AccountReaper({'delay_reaping': '10'}) b = FakeBroker() b.info['delete_timestamp'] = normalize_timestamp(110) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(100) self.assertFalse(r.reap_account(b, 0, None)) b.info['delete_timestamp'] = normalize_timestamp(90) self.assertFalse(r.reap_account(b, 0, None)) # KeyError raised immediately as reap_account tries to get the # account's name to do the reaping. b.info['delete_timestamp'] = normalize_timestamp(89) self.assertRaises(KeyError, r.reap_account, b, 0, None) b.info['delete_timestamp'] = normalize_timestamp(1) self.assertRaises(KeyError, r.reap_account, b, 0, None) finally: reaper.time = time_orig def test_reap_object(self): r = self.init_reaper({}, fakelogger=True) self.amount_fail = 0 self.max_fail = 0 with patch('swift.account.reaper.AccountReaper.get_object_ring', self.fake_object_ring): with patch('swift.account.reaper.direct_delete_object', self.fake_direct_delete_object): r.reap_object('a', 'c', 'partition', cont_nodes, 'o') self.assertEqual(r.stats_objects_deleted, 3) def test_reap_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.amount_fail = 0 self.max_fail = 1 ctx = [patch('swift.account.reaper.AccountReaper.get_object_ring', self.fake_object_ring), patch('swift.account.reaper.direct_delete_object', self.fake_direct_delete_object)] with nested(*ctx): r.reap_object('a', 'c', 'partition', cont_nodes, 'o') self.assertEqual(r.stats_objects_deleted, 1) self.assertEqual(r.stats_objects_remaining, 1) self.assertEqual(r.stats_objects_possibly_remaining, 1) def test_reap_container_get_object_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = True self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 0 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 1) self.assertEqual(r.stats_containers_deleted, 1) def test_reap_container_partial_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 2 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 2) self.assertEqual(r.stats_containers_possibly_remaining, 1) def test_reap_container_full_fail(self): r = self.init_reaper({}, fakelogger=True) self.get_fail = False self.reap_obj_fail = False self.amount_delete_fail = 0 self.max_delete_fail = 3 ctx = [patch('swift.account.reaper.direct_get_container', self.fake_direct_get_container), patch('swift.account.reaper.direct_delete_container', self.fake_direct_delete_container), patch('swift.account.reaper.AccountReaper.get_container_ring', self.fake_container_ring), patch('swift.account.reaper.AccountReaper.reap_object', self.fake_reap_object)] with nested(*ctx): r.reap_container('a', 'partition', acc_nodes, 'c') self.assertEqual(r.logger.inc['return_codes.4'], 3) self.assertEqual(r.stats_containers_remaining, 1) def fake_reap_container(self, *args, **kwargs): self.called_amount += 1 self.r.stats_containers_deleted = 1 self.r.stats_objects_deleted = 1 self.r.stats_containers_remaining = 1 self.r.stats_objects_remaining = 1 self.r.stats_containers_possibly_remaining = 1 self.r.stats_objects_possibly_remaining = 1 def test_reap_account(self): containers = ('c1', 'c2', 'c3', '') broker = FakeAccountBroker(containers) self.called_amount = 0 self.r = r = self.init_reaper({}, fakelogger=True) r.start_time = time.time() ctx = [patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring)] with nested(*ctx): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertEqual(self.called_amount, 4) self.assertEqual(r.logger.msg.find('Completed pass'), 0) self.assertTrue(r.logger.msg.find('1 containers deleted')) self.assertTrue(r.logger.msg.find('1 objects deleted')) self.assertTrue(r.logger.msg.find('1 containers remaining')) self.assertTrue(r.logger.msg.find('1 objects remaining')) self.assertTrue(r.logger.msg.find('1 containers possibly remaining')) self.assertTrue(r.logger.msg.find('1 objects possibly remaining')) def test_reap_account_no_container(self): broker = FakeAccountBroker(tuple()) self.r = r = self.init_reaper({}, fakelogger=True) self.called_amount = 0 r.start_time = time.time() ctx = [patch('swift.account.reaper.AccountReaper.reap_container', self.fake_reap_container), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring)] with nested(*ctx): nodes = r.get_account_ring().get_part_nodes() self.assertTrue(r.reap_account(broker, 'partition', nodes)) self.assertEqual(r.logger.msg.find('Completed pass'), 0) self.assertEqual(self.called_amount, 0) def test_reap_device(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 1) def test_reap_device_with_ts(self): devices = self.prepare_data_dir(ts=True) self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf=conf) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_reap_device_with_not_my_ip(self): devices = self.prepare_data_dir() self.called_amount = 0 conf = {'devices': devices} r = self.init_reaper(conf, myips=['10.10.1.2']) ctx = [patch('swift.account.reaper.AccountBroker', FakeAccountBroker), patch('swift.account.reaper.AccountReaper.get_account_ring', self.fake_account_ring), patch('swift.account.reaper.AccountReaper.reap_account', self.fake_reap_account)] with nested(*ctx): r.reap_device('sda1') self.assertEqual(self.called_amount, 0) def test_run_once(self): def prepare_data_dir(): devices_path = tempfile.mkdtemp() # will be deleted by teardown self.to_delete.append(devices_path) path = os.path.join(devices_path, 'sda1', DATADIR) os.makedirs(path) return devices_path def init_reaper(devices): r = reaper.AccountReaper({'devices': devices}) return r devices = prepare_data_dir() r = init_reaper(devices) with patch('swift.account.reaper.ismount', lambda x: True): with patch( 'swift.account.reaper.AccountReaper.reap_device') as foo: r.run_once() self.assertEqual(foo.called, 1) with patch('swift.account.reaper.ismount', lambda x: False): with patch( 'swift.account.reaper.AccountReaper.reap_device') as foo: r.run_once() self.assertFalse(foo.called) def test_run_forever(self): def fake_sleep(val): self.val = val def fake_random(): return 1 def fake_run_once(): raise Exception('exit') def init_reaper(): r = reaper.AccountReaper({'interval': 1}) r.run_once = fake_run_once return r r = init_reaper() with patch('swift.account.reaper.sleep', fake_sleep): with patch('swift.account.reaper.random.random', fake_random): try: r.run_forever() except Exception, err: pass self.assertEqual(self.val, 1) self.assertEqual(str(err), 'exit') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/account/test_replicator.py0000664000175400017540000000164312323703611023240 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest class TestReplicator(unittest.TestCase): """ swift.account.replicator is currently just a subclass with some class variables overridden, but at least this test stub will ensure proper Python syntax. """ def test_placeholder(self): pass if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/account/test_auditor.py0000664000175400017540000001005112323703611022534 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import mock import time import os import random from tempfile import mkdtemp from shutil import rmtree from swift.account import auditor from test.unit import FakeLogger class FakeAccountBroker(object): def __init__(self, path): self.path = path self.db_file = path self.file = os.path.basename(path) def is_deleted(self): return False def get_info(self): if self.file.startswith('fail'): raise ValueError if self.file.startswith('true'): return 'ok' class TestAuditor(unittest.TestCase): def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_account_auditor') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) fnames = ['true1.db', 'true2.db', 'true3.db', 'fail1.db', 'fail2.db'] for fn in fnames: with open(os.path.join(self.testdir, fn), 'w+') as f: f.write(' ') def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) @mock.patch('swift.account.auditor.AccountBroker', FakeAccountBroker) def test_run_forever(self): sleep_times = random.randint(5, 10) call_times = sleep_times - 1 class FakeTime(object): def __init__(self): self.times = 0 def sleep(self, sec): self.times += 1 if self.times < sleep_times: time.sleep(0.1) else: # stop forever by an error raise ValueError() def time(self): return time.time() conf = {} test_auditor = auditor.AccountAuditor(conf) with mock.patch('swift.account.auditor.time', FakeTime()): def fake_audit_location_generator(*args, **kwargs): files = os.listdir(self.testdir) return [(os.path.join(self.testdir, f), '', '') for f in files] with mock.patch('swift.account.auditor.audit_location_generator', fake_audit_location_generator): self.assertRaises(ValueError, test_auditor.run_forever) self.assertEqual(test_auditor.account_failures, 2 * call_times) self.assertEqual(test_auditor.account_passes, 3 * call_times) @mock.patch('swift.account.auditor.AccountBroker', FakeAccountBroker) def test_run_once(self): conf = {} test_auditor = auditor.AccountAuditor(conf) def fake_audit_location_generator(*args, **kwargs): files = os.listdir(self.testdir) return [(os.path.join(self.testdir, f), '', '') for f in files] with mock.patch('swift.account.auditor.audit_location_generator', fake_audit_location_generator): test_auditor.run_once() self.assertEqual(test_auditor.account_failures, 2) self.assertEqual(test_auditor.account_passes, 3) @mock.patch('swift.account.auditor.AccountBroker', FakeAccountBroker) def test_account_auditor(self): conf = {} test_auditor = auditor.AccountAuditor(conf) files = os.listdir(self.testdir) for f in files: path = os.path.join(self.testdir, f) test_auditor.account_audit(path) self.assertEqual(test_auditor.account_failures, 2) self.assertEqual(test_auditor.account_passes, 3) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/account/test_backend.py0000664000175400017540000005647712323703611022502 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests for swift.account.backend """ import hashlib import unittest from time import sleep, time from uuid import uuid4 from swift.account.backend import AccountBroker from swift.common.utils import normalize_timestamp from swift.common.db import DatabaseConnectionError class TestAccountBroker(unittest.TestCase): """Tests for AccountBroker""" def test_creation(self): # Test AccountBroker.__init__ broker = AccountBroker(':memory:', account='a') self.assertEqual(broker.db_file, ':memory:') try: with broker.get() as conn: pass except DatabaseConnectionError as e: self.assertTrue(hasattr(e, 'path')) self.assertEquals(e.path, ':memory:') self.assertTrue(hasattr(e, 'msg')) self.assertEquals(e.msg, "DB doesn't exist") except Exception as e: self.fail("Unexpected exception raised: %r" % e) else: self.fail("Expected a DatabaseConnectionError exception") broker.initialize(normalize_timestamp('1')) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1) def test_exception(self): # Test AccountBroker throwing a conn away after exception first_conn = None broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: first_conn = conn try: with broker.get() as conn: self.assertEqual(first_conn, conn) raise Exception('OMG') except Exception: pass self.assert_(broker.conn is None) def test_empty(self): # Test AccountBroker.empty broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) self.assert_(broker.empty()) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) self.assert_(not broker.empty()) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) self.assert_(broker.empty()) def test_reclaim(self): broker = AccountBroker(':memory:', account='test_account') broker.initialize(normalize_timestamp('1')) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('c', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(normalize_timestamp(time()), time()) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) # Test reclaim after deletion. Create 3 test containers broker.put_container('x', 0, 0, 0, 0) broker.put_container('y', 0, 0, 0, 0) broker.put_container('z', 0, 0, 0, 0) broker.reclaim(normalize_timestamp(time()), time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assert_(containers is None) # self.assert_(account_name is None) # Now delete the account broker.delete_db(normalize_timestamp(time())) broker.reclaim(normalize_timestamp(time()), time()) # self.assertEqual(len(res), 2) # self.assert_(isinstance(res, tuple)) # containers, account_name = res # self.assertEqual(account_name, 'test_account') # self.assertEqual(len(containers), 3) # self.assert_('x' in containers) # self.assert_('y' in containers) # self.assert_('z' in containers) # self.assert_('a' not in containers) def test_delete_container(self): # Test AccountBroker.delete_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('o', normalize_timestamp(time()), 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 1) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.put_container('o', 0, normalize_timestamp(time()), 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 0").fetchone()[0], 0) self.assertEqual(conn.execute( "SELECT count(*) FROM container " "WHERE deleted = 1").fetchone()[0], 1) def test_put_container(self): # Test AccountBroker.put_container broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) # Create initial container timestamp = normalize_timestamp(time()) broker.put_container('"{}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Reput same event broker.put_container('"{}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old event otimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{}"', otimestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put old delete event dtimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_container('"{}"', 0, dtimestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], dtimestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) # Put new delete event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{}"', 0, timestamp, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT delete_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 1) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_container('"{}"', timestamp, 0, 0, 0) with broker.get() as conn: self.assertEqual(conn.execute( "SELECT name FROM container").fetchone()[0], '"{}"') self.assertEqual(conn.execute( "SELECT put_timestamp FROM container").fetchone()[0], timestamp) self.assertEqual(conn.execute( "SELECT deleted FROM container").fetchone()[0], 0) def test_get_info(self): # Test AccountBroker.get_info broker = AccountBroker(':memory:', account='test1') broker.initialize(normalize_timestamp('1')) info = broker.get_info() self.assertEqual(info['account'], 'test1') self.assertEqual(info['hash'], '00000000000000000000000000000000') info = broker.get_info() self.assertEqual(info['container_count'], 0) broker.put_container('c1', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c2', normalize_timestamp(time()), 0, 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 2) sleep(.00001) broker.put_container('c1', 0, normalize_timestamp(time()), 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 1) sleep(.00001) broker.put_container('c2', 0, normalize_timestamp(time()), 0, 0) info = broker.get_info() self.assertEqual(info['container_count'], 0) def test_list_containers_iter(self): # Test AccountBroker.list_containers_iter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) for cont1 in xrange(4): for cont2 in xrange(125): broker.put_container('%d-%04d' % (cont1, cont2), normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('2-0051-%04d' % cont, normalize_timestamp(time()), 0, 0, 0) for cont in xrange(125): broker.put_container('3-%04d-0049' % cont, normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(100, '', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = broker.list_containers_iter(100, '', '0-0050', None, '') self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = broker.list_containers_iter(100, '0-0099', None, None, '') self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = broker.list_containers_iter(55, '1-0074', None, None, '') self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = broker.list_containers_iter(10, '', None, '0-01', '') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-01', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = broker.list_containers_iter(10, '', None, '0-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = broker.list_containers_iter(10, '', None, '', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['0-', '1-', '2-', '3-']) listing = broker.list_containers_iter(10, '2-', None, None, '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['3-']) listing = broker.list_containers_iter(10, '', None, '2', '-') self.assertEqual(len(listing), 1) self.assertEqual([row[0] for row in listing], ['2-']) listing = broker.list_containers_iter(10, '2-0050', None, '2-', '-') self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = broker.list_containers_iter(10, '3-0045', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050']) broker.put_container('3-0049-', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(10, '3-0048', None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049']) listing = broker.list_containers_iter(10, '3-0048', None, '3-', '-') self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053']) listing = broker.list_containers_iter(10, None, None, '3-0049-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['3-0049-', '3-0049-0049']) def test_double_check_trailing_delimiter(self): # Test AccountBroker.list_containers_iter for an # account that has an odd container with a trailing delimiter broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('a-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b-b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) listing = broker.list_containers_iter(15, None, None, None, None) self.assertEqual(len(listing), 10) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'a-a', 'a-a-a', 'a-a-b', 'a-b', 'b', 'b-a', 'b-b', 'c']) listing = broker.list_containers_iter(15, None, None, '', '-') self.assertEqual(len(listing), 5) self.assertEqual([row[0] for row in listing], ['a', 'a-', 'b', 'b-', 'c']) listing = broker.list_containers_iter(15, None, None, 'a-', '-') self.assertEqual(len(listing), 4) self.assertEqual([row[0] for row in listing], ['a-', 'a-a', 'a-a-', 'a-b']) listing = broker.list_containers_iter(15, None, None, 'b-', '-') self.assertEqual(len(listing), 2) self.assertEqual([row[0] for row in listing], ['b-a', 'b-b']) def test_chexor(self): broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) broker.put_container('a', normalize_timestamp(1), normalize_timestamp(0), 0, 0) broker.put_container('b', normalize_timestamp(2), normalize_timestamp(0), 0, 0) hasha = hashlib.md5( '%s-%s' % ('a', '0000000001.00000-0000000000.00000-0-0') ).digest() hashb = hashlib.md5( '%s-%s' % ('b', '0000000002.00000-0000000000.00000-0-0') ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) broker.put_container('b', normalize_timestamp(3), normalize_timestamp(0), 0, 0) hashb = hashlib.md5( '%s-%s' % ('b', '0000000003.00000-0000000000.00000-0-0') ).digest() hashc = \ ''.join(('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEqual(broker.get_info()['hash'], hashc) def test_merge_items(self): broker1 = AccountBroker(':memory:', account='a') broker1.initialize(normalize_timestamp('1')) broker2 = AccountBroker(':memory:', account='a') broker2.initialize(normalize_timestamp('1')) broker1.put_container('a', normalize_timestamp(1), 0, 0, 0) broker1.put_container('b', normalize_timestamp(2), 0, 0, 0) id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 2) self.assertEqual(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_container('c', normalize_timestamp(3), 0, 0, 0) broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEqual(len(items), 3) self.assertEqual(['a', 'b', 'c'], sorted([rec['name'] for rec in items])) def premetadata_create_account_stat_table(self, conn, put_timestamp): """ Copied from AccountBroker before the metadata column was added; used for testing with TestAccountBrokerBeforeMetadata. Create account_stat table which is specific to the account DB. :param conn: DB connection object :param put_timestamp: put timestamp """ conn.executescript(''' CREATE TABLE account_stat ( account TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', container_count INTEGER, object_count INTEGER DEFAULT 0, bytes_used INTEGER DEFAULT 0, hash TEXT default '00000000000000000000000000000000', id TEXT, status TEXT DEFAULT '', status_changed_at TEXT DEFAULT '0' ); INSERT INTO account_stat (container_count) VALUES (0); ''') conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ? ''', (self.account, normalize_timestamp(time()), str(uuid4()), put_timestamp)) class TestAccountBrokerBeforeMetadata(TestAccountBroker): """ Tests for AccountBroker against databases created before the metadata column was added. """ def setUp(self): self._imported_create_account_stat_table = \ AccountBroker.create_account_stat_table AccountBroker.create_account_stat_table = \ premetadata_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) exc = None with broker.get() as conn: try: conn.execute('SELECT metadata FROM account_stat') except BaseException as err: exc = err self.assert_('no such column: metadata' in str(exc)) def tearDown(self): AccountBroker.create_account_stat_table = \ self._imported_create_account_stat_table broker = AccountBroker(':memory:', account='a') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('SELECT metadata FROM account_stat') swift-1.13.1/test/unit/account/__init__.py0000664000175400017540000000000012323703611021556 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/0000775000175400017540000000000012323703665017324 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/test_memcached.py0000664000175400017540000005400512323703611022636 0ustar jenkinsjenkins00000000000000# -*- coding:utf-8 -*- # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.common.utils""" from collections import defaultdict import logging import socket import time import unittest from uuid import uuid4 from eventlet import GreenPool, sleep, Queue from eventlet.pools import Pool from swift.common import memcached from mock import patch, MagicMock from test.unit import NullLoggingHandler class MockedMemcachePool(memcached.MemcacheConnPool): def __init__(self, mocks): Pool.__init__(self, max_size=2) self.mocks = mocks # setting this for the eventlet workaround in the MemcacheConnPool self._parent_class_getter = super(memcached.MemcacheConnPool, self).get def create(self): return self.mocks.pop(0) class ExplodingMockMemcached(object): exploded = False def sendall(self, string): self.exploded = True raise socket.error() def readline(self): self.exploded = True raise socket.error() def read(self, size): self.exploded = True raise socket.error() def close(self): pass class MockMemcached(object): def __init__(self): self.inbuf = '' self.outbuf = '' self.cache = {} self.down = False self.exc_on_delete = False self.read_return_none = False self.close_called = False def sendall(self, string): if self.down: raise Exception('mock is down') self.inbuf += string while '\n' in self.inbuf: cmd, self.inbuf = self.inbuf.split('\n', 1) parts = cmd.split() if parts[0].lower() == 'set': self.cache[parts[1]] = parts[2], parts[3], \ self.inbuf[:int(parts[4])] self.inbuf = self.inbuf[int(parts[4]) + 2:] if len(parts) < 6 or parts[5] != 'noreply': self.outbuf += 'STORED\r\n' elif parts[0].lower() == 'add': value = self.inbuf[:int(parts[4])] self.inbuf = self.inbuf[int(parts[4]) + 2:] if parts[1] in self.cache: if len(parts) < 6 or parts[5] != 'noreply': self.outbuf += 'NOT_STORED\r\n' else: self.cache[parts[1]] = parts[2], parts[3], value if len(parts) < 6 or parts[5] != 'noreply': self.outbuf += 'STORED\r\n' elif parts[0].lower() == 'delete': if self.exc_on_delete: raise Exception('mock is has exc_on_delete set') if parts[1] in self.cache: del self.cache[parts[1]] if 'noreply' not in parts: self.outbuf += 'DELETED\r\n' elif 'noreply' not in parts: self.outbuf += 'NOT_FOUND\r\n' elif parts[0].lower() == 'get': for key in parts[1:]: if key in self.cache: val = self.cache[key] self.outbuf += 'VALUE %s %s %s\r\n' % ( key, val[0], len(val[2])) self.outbuf += val[2] + '\r\n' self.outbuf += 'END\r\n' elif parts[0].lower() == 'incr': if parts[1] in self.cache: val = list(self.cache[parts[1]]) val[2] = str(int(val[2]) + int(parts[2])) self.cache[parts[1]] = val self.outbuf += str(val[2]) + '\r\n' else: self.outbuf += 'NOT_FOUND\r\n' elif parts[0].lower() == 'decr': if parts[1] in self.cache: val = list(self.cache[parts[1]]) if int(val[2]) - int(parts[2]) > 0: val[2] = str(int(val[2]) - int(parts[2])) else: val[2] = '0' self.cache[parts[1]] = val self.outbuf += str(val[2]) + '\r\n' else: self.outbuf += 'NOT_FOUND\r\n' def readline(self): if self.read_return_none: return None if self.down: raise Exception('mock is down') if '\n' in self.outbuf: response, self.outbuf = self.outbuf.split('\n', 1) return response + '\n' def read(self, size): if self.down: raise Exception('mock is down') if len(self.outbuf) >= size: response = self.outbuf[:size] self.outbuf = self.outbuf[size:] return response def close(self): self.close_called = True pass class TestMemcached(unittest.TestCase): """Tests for swift.common.memcached""" def test_get_conns(self): sock1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock1.bind(('127.0.0.1', 0)) sock1.listen(1) sock1ipport = '%s:%s' % sock1.getsockname() sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock2.bind(('127.0.0.1', 0)) sock2.listen(1) orig_port = memcached.DEFAULT_MEMCACHED_PORT try: sock2ip, memcached.DEFAULT_MEMCACHED_PORT = sock2.getsockname() sock2ipport = '%s:%s' % (sock2ip, memcached.DEFAULT_MEMCACHED_PORT) # We're deliberately using sock2ip (no port) here to test that the # default port is used. memcache_client = memcached.MemcacheRing([sock1ipport, sock2ip]) one = two = True while one or two: # Run until we match hosts one and two key = uuid4().hex for conn in memcache_client._get_conns(key): peeripport = '%s:%s' % conn[2].getpeername() self.assert_(peeripport in (sock1ipport, sock2ipport)) if peeripport == sock1ipport: one = False if peeripport == sock2ipport: two = False finally: memcached.DEFAULT_MEMCACHED_PORT = orig_port def test_set_get(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.set('some_key', [1, 2, 3]) self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) self.assertEquals(mock.cache.values()[0][1], '0') memcache_client.set('some_key', [4, 5, 6]) self.assertEquals(memcache_client.get('some_key'), [4, 5, 6]) memcache_client.set('some_key', ['simple str', 'utf8 str éà']) # As per http://wiki.openstack.org/encoding, # we should expect to have unicode self.assertEquals( memcache_client.get('some_key'), ['simple str', u'utf8 str éà']) self.assert_(float(mock.cache.values()[0][1]) == 0) memcache_client.set('some_key', [1, 2, 3], timeout=10) self.assertEquals(mock.cache.values()[0][1], '10') memcache_client.set('some_key', [1, 2, 3], time=20) self.assertEquals(mock.cache.values()[0][1], '20') sixtydays = 60 * 24 * 60 * 60 esttimeout = time.time() + sixtydays memcache_client.set('some_key', [1, 2, 3], timeout=sixtydays) self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) memcache_client.set('some_key', [1, 2, 3], time=sixtydays) self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) def test_incr(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) self.assertEquals(memcache_client.incr('some_key', delta=5), 5) self.assertEquals(memcache_client.get('some_key'), '5') self.assertEquals(memcache_client.incr('some_key', delta=5), 10) self.assertEquals(memcache_client.get('some_key'), '10') self.assertEquals(memcache_client.incr('some_key', delta=1), 11) self.assertEquals(memcache_client.get('some_key'), '11') self.assertEquals(memcache_client.incr('some_key', delta=-5), 6) self.assertEquals(memcache_client.get('some_key'), '6') self.assertEquals(memcache_client.incr('some_key', delta=-15), 0) self.assertEquals(memcache_client.get('some_key'), '0') mock.read_return_none = True self.assertRaises(memcached.MemcacheConnectionError, memcache_client.incr, 'some_key', delta=-15) self.assertTrue(mock.close_called) def test_incr_w_timeout(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.incr('some_key', delta=5, time=55) self.assertEquals(memcache_client.get('some_key'), '5') self.assertEquals(mock.cache.values()[0][1], '55') memcache_client.delete('some_key') self.assertEquals(memcache_client.get('some_key'), None) fiftydays = 50 * 24 * 60 * 60 esttimeout = time.time() + fiftydays memcache_client.incr('some_key', delta=5, time=fiftydays) self.assertEquals(memcache_client.get('some_key'), '5') self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) memcache_client.delete('some_key') self.assertEquals(memcache_client.get('some_key'), None) memcache_client.incr('some_key', delta=5) self.assertEquals(memcache_client.get('some_key'), '5') self.assertEquals(mock.cache.values()[0][1], '0') memcache_client.incr('some_key', delta=5, time=55) self.assertEquals(memcache_client.get('some_key'), '10') self.assertEquals(mock.cache.values()[0][1], '0') def test_decr(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) self.assertEquals(memcache_client.decr('some_key', delta=5), 0) self.assertEquals(memcache_client.get('some_key'), '0') self.assertEquals(memcache_client.incr('some_key', delta=15), 15) self.assertEquals(memcache_client.get('some_key'), '15') self.assertEquals(memcache_client.decr('some_key', delta=4), 11) self.assertEquals(memcache_client.get('some_key'), '11') self.assertEquals(memcache_client.decr('some_key', delta=15), 0) self.assertEquals(memcache_client.get('some_key'), '0') mock.read_return_none = True self.assertRaises(memcached.MemcacheConnectionError, memcache_client.decr, 'some_key', delta=15) def test_retry(self): logging.getLogger().addHandler(NullLoggingHandler()) memcache_client = memcached.MemcacheRing( ['1.2.3.4:11211', '1.2.3.5:11211']) mock1 = ExplodingMockMemcached() mock2 = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock2, mock2)]) memcache_client._client_cache['1.2.3.5:11211'] = MockedMemcachePool( [(mock1, mock1)]) memcache_client.set('some_key', [1, 2, 3]) self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) self.assertEquals(mock1.exploded, True) def test_delete(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.set('some_key', [1, 2, 3]) self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) memcache_client.delete('some_key') self.assertEquals(memcache_client.get('some_key'), None) def test_multi(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211']) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.set_multi( {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key') self.assertEquals( memcache_client.get_multi(('some_key2', 'some_key1'), 'multi_key'), [[4, 5, 6], [1, 2, 3]]) self.assertEquals(mock.cache.values()[0][1], '0') self.assertEquals(mock.cache.values()[1][1], '0') memcache_client.set_multi( {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key', timeout=10) self.assertEquals(mock.cache.values()[0][1], '10') self.assertEquals(mock.cache.values()[1][1], '10') memcache_client.set_multi( {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key', time=20) self.assertEquals(mock.cache.values()[0][1], '20') self.assertEquals(mock.cache.values()[1][1], '20') fortydays = 50 * 24 * 60 * 60 esttimeout = time.time() + fortydays memcache_client.set_multi( {'some_key1': [1, 2, 3], 'some_key2': [4, 5, 6]}, 'multi_key', timeout=fortydays) self.assert_(-1 <= float(mock.cache.values()[0][1]) - esttimeout <= 1) self.assert_(-1 <= float(mock.cache.values()[1][1]) - esttimeout <= 1) self.assertEquals(memcache_client.get_multi( ('some_key2', 'some_key1', 'not_exists'), 'multi_key'), [[4, 5, 6], [1, 2, 3], None]) def test_serialization(self): memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'], allow_pickle=True) mock = MockMemcached() memcache_client._client_cache['1.2.3.4:11211'] = MockedMemcachePool( [(mock, mock)] * 2) memcache_client.set('some_key', [1, 2, 3]) self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) memcache_client._allow_pickle = False memcache_client._allow_unpickle = True self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) memcache_client._allow_unpickle = False self.assertEquals(memcache_client.get('some_key'), None) memcache_client.set('some_key', [1, 2, 3]) self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) memcache_client._allow_unpickle = True self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) memcache_client._allow_pickle = True self.assertEquals(memcache_client.get('some_key'), [1, 2, 3]) def test_connection_pooling(self): with patch('swift.common.memcached.socket') as mock_module: # patch socket, stub socket.socket, mock sock mock_sock = mock_module.socket.return_value # track clients waiting for connections connected = [] connections = Queue() errors = [] def wait_connect(addr): connected.append(addr) sleep(0.1) # yield val = connections.get() if val is not None: errors.append(val) mock_sock.connect = wait_connect memcache_client = memcached.MemcacheRing(['1.2.3.4:11211'], connect_timeout=10) # sanity self.assertEquals(1, len(memcache_client._client_cache)) for server, pool in memcache_client._client_cache.items(): self.assertEqual(2, pool.max_size) # make 10 requests "at the same time" p = GreenPool() for i in range(10): p.spawn(memcache_client.set, 'key', 'value') for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # give out a connection connections.put(None) # at this point, only one connection should have actually been # created, the other is in the creation step, and the rest of the # clients are not attempting to connect. we let this play out a # bit to verify. for i in range(3): sleep(0.1) self.assertEqual(2, len(connected)) # finish up, this allows the final connection to be created, so # that all the other clients can use the two existing connections # and no others will be created. connections.put(None) connections.put('nono') self.assertEqual(2, len(connected)) p.waitall() self.assertEqual(2, len(connected)) self.assertEqual(0, len(errors), "A client was allowed a third connection") connections.get_nowait() self.assertTrue(connections.empty()) # Ensure we exercise the backported-for-pre-eventlet-version-0.9.17 get() # code, even if the executing eventlet's version is already newer. @patch.object(memcached, 'eventlet_version', '0.9.16') def test_connection_pooling_pre_0_9_17(self): with patch('swift.common.memcached.socket') as mock_module: connected = [] count = [0] def _slow_yielding_connector(addr): count[0] += 1 if count[0] % 3 == 0: raise ValueError('whoops!') sleep(0.1) connected.append(addr) mock_module.socket.return_value.connect.side_effect = \ _slow_yielding_connector # If POOL_SIZE is not small enough relative to USER_COUNT, the # "free_items" business in the eventlet.pools.Pool will cause # spurious failures below. I found these values to work well on a # VM running in VirtualBox on a late 2013 Retina MacbookPro: POOL_SIZE = 5 USER_COUNT = 50 pool = memcached.MemcacheConnPool('1.2.3.4:11211', size=POOL_SIZE, connect_timeout=10) self.assertEqual(POOL_SIZE, pool.max_size) def _user(): got = None while not got: try: got = pool.get() except: # noqa pass pool.put(got) # make a bunch of requests "at the same time" p = GreenPool() for i in range(USER_COUNT): p.spawn(_user) p.waitall() # If the except block after the "created = self.create()" call # doesn't correctly decrement self.current_size, this test will # fail by having some number less than POOL_SIZE connections (in my # testing, anyway). self.assertEqual(POOL_SIZE, len(connected)) # Subsequent requests should get and use the existing # connections, not creating any more. for i in range(USER_COUNT): p.spawn(_user) p.waitall() self.assertEqual(POOL_SIZE, len(connected)) def test_connection_pool_timeout(self): orig_conn_pool = memcached.MemcacheConnPool try: connections = defaultdict(Queue) pending = defaultdict(int) served = defaultdict(int) class MockConnectionPool(orig_conn_pool): def get(self): pending[self.server] += 1 conn = connections[self.server].get() pending[self.server] -= 1 return conn def put(self, *args, **kwargs): connections[self.server].put(*args, **kwargs) served[self.server] += 1 memcached.MemcacheConnPool = MockConnectionPool memcache_client = memcached.MemcacheRing(['1.2.3.4:11211', '1.2.3.5:11211'], io_timeout=0.5, pool_timeout=0.1) # Hand out a couple slow connections to 1.2.3.5, leaving 1.2.3.4 # fast. All ten (10) clients should try to talk to .5 first, and # then move on to .4, and we'll assert all that below. mock_conn = MagicMock(), MagicMock() mock_conn[1].sendall = lambda x: sleep(0.2) connections['1.2.3.5:11211'].put(mock_conn) connections['1.2.3.5:11211'].put(mock_conn) mock_conn = MagicMock(), MagicMock() connections['1.2.3.4:11211'].put(mock_conn) connections['1.2.3.4:11211'].put(mock_conn) p = GreenPool() for i in range(10): p.spawn(memcache_client.set, 'key', 'value') # Wait for the dust to settle. p.waitall() self.assertEqual(pending['1.2.3.5:11211'], 8) self.assertEqual(len(memcache_client._errors['1.2.3.5:11211']), 8) self.assertEqual(served['1.2.3.5:11211'], 2) self.assertEqual(pending['1.2.3.4:11211'], 0) self.assertEqual(len(memcache_client._errors['1.2.3.4:11211']), 0) self.assertEqual(served['1.2.3.4:11211'], 8) # and we never got more put in that we gave out self.assertEqual(connections['1.2.3.5:11211'].qsize(), 2) self.assertEqual(connections['1.2.3.4:11211'].qsize(), 2) finally: memcached.MemcacheConnPool = orig_conn_pool if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_swob.py0000664000175400017540000016346112323703614021714 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. "Tests for swift.common.swob" import unittest import datetime import re import time from StringIO import StringIO from urllib import quote import swift.common.swob class TestHeaderEnvironProxy(unittest.TestCase): def test_proxy(self): environ = {} proxy = swift.common.swob.HeaderEnvironProxy(environ) proxy['Content-Length'] = 20 proxy['Content-Type'] = 'text/plain' proxy['Something-Else'] = 'somevalue' self.assertEquals( proxy.environ, {'CONTENT_LENGTH': '20', 'CONTENT_TYPE': 'text/plain', 'HTTP_SOMETHING_ELSE': 'somevalue'}) self.assertEquals(proxy['content-length'], '20') self.assertEquals(proxy['content-type'], 'text/plain') self.assertEquals(proxy['something-else'], 'somevalue') def test_del(self): environ = {} proxy = swift.common.swob.HeaderEnvironProxy(environ) proxy['Content-Length'] = 20 proxy['Content-Type'] = 'text/plain' proxy['Something-Else'] = 'somevalue' del proxy['Content-Length'] del proxy['Content-Type'] del proxy['Something-Else'] self.assertEquals(proxy.environ, {}) def test_contains(self): environ = {} proxy = swift.common.swob.HeaderEnvironProxy(environ) proxy['Content-Length'] = 20 proxy['Content-Type'] = 'text/plain' proxy['Something-Else'] = 'somevalue' self.assert_('content-length' in proxy) self.assert_('content-type' in proxy) self.assert_('something-else' in proxy) def test_keys(self): environ = {} proxy = swift.common.swob.HeaderEnvironProxy(environ) proxy['Content-Length'] = 20 proxy['Content-Type'] = 'text/plain' proxy['Something-Else'] = 'somevalue' self.assertEquals( set(proxy.keys()), set(('Content-Length', 'Content-Type', 'Something-Else'))) class TestHeaderKeyDict(unittest.TestCase): def test_case_insensitive(self): headers = swift.common.swob.HeaderKeyDict() headers['Content-Length'] = 0 headers['CONTENT-LENGTH'] = 10 headers['content-length'] = 20 self.assertEquals(headers['Content-Length'], '20') self.assertEquals(headers['content-length'], '20') self.assertEquals(headers['CONTENT-LENGTH'], '20') def test_setdefault(self): headers = swift.common.swob.HeaderKeyDict() # it gets set headers.setdefault('x-rubber-ducky', 'the one') self.assertEquals(headers['X-Rubber-Ducky'], 'the one') # it has the right return value ret = headers.setdefault('x-boat', 'dinghy') self.assertEquals(ret, 'dinghy') ret = headers.setdefault('x-boat', 'yacht') self.assertEquals(ret, 'dinghy') # shouldn't crash headers.setdefault('x-sir-not-appearing-in-this-request', None) def test_del_contains(self): headers = swift.common.swob.HeaderKeyDict() headers['Content-Length'] = 0 self.assert_('Content-Length' in headers) del headers['Content-Length'] self.assert_('Content-Length' not in headers) def test_update(self): headers = swift.common.swob.HeaderKeyDict() headers.update({'Content-Length': '0'}) headers.update([('Content-Type', 'text/plain')]) self.assertEquals(headers['Content-Length'], '0') self.assertEquals(headers['Content-Type'], 'text/plain') def test_get(self): headers = swift.common.swob.HeaderKeyDict() headers['content-length'] = 20 self.assertEquals(headers.get('CONTENT-LENGTH'), '20') self.assertEquals(headers.get('something-else'), None) self.assertEquals(headers.get('something-else', True), True) def test_keys(self): headers = swift.common.swob.HeaderKeyDict() headers['content-length'] = 20 headers['cOnTent-tYpe'] = 'text/plain' headers['SomeThing-eLse'] = 'somevalue' self.assertEquals( set(headers.keys()), set(('Content-Length', 'Content-Type', 'Something-Else'))) class TestRange(unittest.TestCase): def test_range(self): range = swift.common.swob.Range('bytes=1-7') self.assertEquals(range.ranges[0], (1, 7)) def test_upsidedown_range(self): range = swift.common.swob.Range('bytes=5-10') self.assertEquals(range.ranges_for_length(2), []) def test_str(self): for range_str in ('bytes=1-7', 'bytes=1-', 'bytes=-1', 'bytes=1-7,9-12', 'bytes=-7,9-'): range = swift.common.swob.Range(range_str) self.assertEquals(str(range), range_str) def test_ranges_for_length(self): range = swift.common.swob.Range('bytes=1-7') self.assertEquals(range.ranges_for_length(10), [(1, 8)]) self.assertEquals(range.ranges_for_length(5), [(1, 5)]) self.assertEquals(range.ranges_for_length(None), None) def test_ranges_for_large_length(self): range = swift.common.swob.Range('bytes=-1000000000000000000000000000') self.assertEquals(range.ranges_for_length(100), [(0, 100)]) def test_ranges_for_length_no_end(self): range = swift.common.swob.Range('bytes=1-') self.assertEquals(range.ranges_for_length(10), [(1, 10)]) self.assertEquals(range.ranges_for_length(5), [(1, 5)]) self.assertEquals(range.ranges_for_length(None), None) # This used to freak out: range = swift.common.swob.Range('bytes=100-') self.assertEquals(range.ranges_for_length(5), []) self.assertEquals(range.ranges_for_length(None), None) range = swift.common.swob.Range('bytes=4-6,100-') self.assertEquals(range.ranges_for_length(5), [(4, 5)]) def test_ranges_for_length_no_start(self): range = swift.common.swob.Range('bytes=-7') self.assertEquals(range.ranges_for_length(10), [(3, 10)]) self.assertEquals(range.ranges_for_length(5), [(0, 5)]) self.assertEquals(range.ranges_for_length(None), None) range = swift.common.swob.Range('bytes=4-6,-100') self.assertEquals(range.ranges_for_length(5), [(4, 5), (0, 5)]) def test_ranges_for_length_multi(self): range = swift.common.swob.Range('bytes=-20,4-,30-150,-10') # the length of the ranges should be 4 self.assertEquals(len(range.ranges_for_length(200)), 4) # the actual length less than any of the range self.assertEquals(range.ranges_for_length(90), [(70, 90), (4, 90), (30, 90), (80, 90)]) # the actual length greater than any of the range self.assertEquals(range.ranges_for_length(200), [(180, 200), (4, 200), (30, 151), (190, 200)]) self.assertEquals(range.ranges_for_length(None), None) def test_ranges_for_length_edges(self): range = swift.common.swob.Range('bytes=0-1, -7') self.assertEquals(range.ranges_for_length(10), [(0, 2), (3, 10)]) range = swift.common.swob.Range('bytes=-7, 0-1') self.assertEquals(range.ranges_for_length(10), [(3, 10), (0, 2)]) range = swift.common.swob.Range('bytes=-7, 0-1') self.assertEquals(range.ranges_for_length(5), [(0, 5), (0, 2)]) def test_range_invalid_syntax(self): def _check_invalid_range(range_value): try: swift.common.swob.Range(range_value) return False except ValueError: return True """ All the following cases should result ValueError exception 1. value not starts with bytes= 2. range value start is greater than the end, eg. bytes=5-3 3. range does not have start or end, eg. bytes=- 4. range does not have hyphen, eg. bytes=45 5. range value is non numeric 6. any combination of the above """ self.assert_(_check_invalid_range('nonbytes=foobar,10-2')) self.assert_(_check_invalid_range('bytes=5-3')) self.assert_(_check_invalid_range('bytes=-')) self.assert_(_check_invalid_range('bytes=45')) self.assert_(_check_invalid_range('bytes=foo-bar,3-5')) self.assert_(_check_invalid_range('bytes=4-10,45')) self.assert_(_check_invalid_range('bytes=foobar,3-5')) self.assert_(_check_invalid_range('bytes=nonumber-5')) self.assert_(_check_invalid_range('bytes=nonumber')) class TestMatch(unittest.TestCase): def test_match(self): match = swift.common.swob.Match('"a", "b"') self.assertEquals(match.tags, set(('a', 'b'))) self.assert_('a' in match) self.assert_('b' in match) self.assert_('c' not in match) def test_match_star(self): match = swift.common.swob.Match('"a", "*"') self.assert_('a' in match) self.assert_('b' in match) self.assert_('c' in match) def test_match_noquote(self): match = swift.common.swob.Match('a, b') self.assertEquals(match.tags, set(('a', 'b'))) self.assert_('a' in match) self.assert_('b' in match) self.assert_('c' not in match) class TestAccept(unittest.TestCase): def test_accept_json(self): for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/json;q=1.0', 'application/*', 'text/*,application/json', 'application/*,text/*', 'application/json,text/xml'): acc = swift.common.swob.Accept(accept) match = acc.best_match(['text/plain', 'application/json', 'application/xml', 'text/xml']) self.assertEquals(match, 'application/json') def test_accept_plain(self): for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', '*/*;q=0.9,application/xml;q=0.8', '*/*', 'text/plain,application/xml'): acc = swift.common.swob.Accept(accept) match = acc.best_match(['text/plain', 'application/json', 'application/xml', 'text/xml']) self.assertEquals(match, 'text/plain') def test_accept_xml(self): for accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/xml;q=1.0', 'application/xml;charset=UTF-8', 'application/xml;charset=UTF-8;qws="quoted with space"', 'application/xml; q=0.99 ; qws="quoted with space"'): acc = swift.common.swob.Accept(accept) match = acc.best_match(['text/plain', 'application/xml', 'text/xml']) self.assertEquals(match, 'application/xml') def test_accept_invalid(self): for accept in ('*', 'text/plain,,', 'some stuff', 'application/xml;q=1.0;q=1.1', 'text/plain,*', 'text /plain', 'text\x7f/plain', 'text/plain;a=b=c', 'text/plain;q=1;q=2', 'text/plain; ubq="unbalanced " quotes"'): acc = swift.common.swob.Accept(accept) match = acc.best_match(['text/plain', 'application/xml', 'text/xml']) self.assertEquals(match, None) def test_repr(self): acc = swift.common.swob.Accept("application/json") self.assertEquals(repr(acc), "application/json") class TestRequest(unittest.TestCase): def test_blank(self): req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}, body='hi') self.assertEquals(req.path_info, '/') self.assertEquals(req.body, 'hi') self.assertEquals(req.headers['Content-Type'], 'text/plain') self.assertEquals(req.method, 'POST') def test_blank_req_environ_property_args(self): blank = swift.common.swob.Request.blank req = blank('/', method='PATCH') self.assertEquals(req.method, 'PATCH') self.assertEquals(req.environ['REQUEST_METHOD'], 'PATCH') req = blank('/', referer='http://example.com') self.assertEquals(req.referer, 'http://example.com') self.assertEquals(req.referrer, 'http://example.com') self.assertEquals(req.environ['HTTP_REFERER'], 'http://example.com') self.assertEquals(req.headers['Referer'], 'http://example.com') req = blank('/', script_name='/application') self.assertEquals(req.script_name, '/application') self.assertEquals(req.environ['SCRIPT_NAME'], '/application') req = blank('/', host='www.example.com') self.assertEquals(req.host, 'www.example.com') self.assertEquals(req.environ['HTTP_HOST'], 'www.example.com') self.assertEquals(req.headers['Host'], 'www.example.com') req = blank('/', remote_addr='127.0.0.1') self.assertEquals(req.remote_addr, '127.0.0.1') self.assertEquals(req.environ['REMOTE_ADDR'], '127.0.0.1') req = blank('/', remote_user='username') self.assertEquals(req.remote_user, 'username') self.assertEquals(req.environ['REMOTE_USER'], 'username') req = blank('/', user_agent='curl/7.22.0 (x86_64-pc-linux-gnu)') self.assertEquals(req.user_agent, 'curl/7.22.0 (x86_64-pc-linux-gnu)') self.assertEquals(req.environ['HTTP_USER_AGENT'], 'curl/7.22.0 (x86_64-pc-linux-gnu)') self.assertEquals(req.headers['User-Agent'], 'curl/7.22.0 (x86_64-pc-linux-gnu)') req = blank('/', query_string='a=b&c=d') self.assertEquals(req.query_string, 'a=b&c=d') self.assertEquals(req.environ['QUERY_STRING'], 'a=b&c=d') req = blank('/', if_match='*') self.assertEquals(req.environ['HTTP_IF_MATCH'], '*') self.assertEquals(req.headers['If-Match'], '*') # multiple environ property kwargs req = blank('/', method='PATCH', referer='http://example.com', script_name='/application', host='www.example.com', remote_addr='127.0.0.1', remote_user='username', user_agent='curl/7.22.0 (x86_64-pc-linux-gnu)', query_string='a=b&c=d', if_match='*') self.assertEquals(req.method, 'PATCH') self.assertEquals(req.referer, 'http://example.com') self.assertEquals(req.script_name, '/application') self.assertEquals(req.host, 'www.example.com') self.assertEquals(req.remote_addr, '127.0.0.1') self.assertEquals(req.remote_user, 'username') self.assertEquals(req.user_agent, 'curl/7.22.0 (x86_64-pc-linux-gnu)') self.assertEquals(req.query_string, 'a=b&c=d') self.assertEquals(req.environ['QUERY_STRING'], 'a=b&c=d') def test_invalid_req_environ_property_args(self): # getter only property try: swift.common.swob.Request.blank('/', params={'a': 'b'}) except TypeError as e: self.assertEquals("got unexpected keyword argument 'params'", str(e)) else: self.assert_(False, "invalid req_environ_property " "didn't raise error!") # regular attribute try: swift.common.swob.Request.blank('/', _params_cache={'a': 'b'}) except TypeError as e: self.assertEquals("got unexpected keyword " "argument '_params_cache'", str(e)) else: self.assert_(False, "invalid req_environ_property " "didn't raise error!") # non-existant attribute try: swift.common.swob.Request.blank('/', params_cache={'a': 'b'}) except TypeError as e: self.assertEquals("got unexpected keyword " "argument 'params_cache'", str(e)) else: self.assert_(False, "invalid req_environ_property " "didn't raise error!") # method try: swift.common.swob.Request.blank( '/', as_referer='GET http://example.com') except TypeError as e: self.assertEquals("got unexpected keyword " "argument 'as_referer'", str(e)) else: self.assert_(False, "invalid req_environ_property " "didn't raise error!") def test_blank_path_info_precedence(self): blank = swift.common.swob.Request.blank req = blank('/a') self.assertEquals(req.path_info, '/a') req = blank('/a', environ={'PATH_INFO': '/a/c'}) self.assertEquals(req.path_info, '/a/c') req = blank('/a', environ={'PATH_INFO': '/a/c'}, path_info='/a/c/o') self.assertEquals(req.path_info, '/a/c/o') req = blank('/a', path_info='/a/c/o') self.assertEquals(req.path_info, '/a/c/o') def test_blank_body_precedence(self): req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('')}, headers={'Content-Type': 'text/plain'}, body='hi') self.assertEquals(req.path_info, '/') self.assertEquals(req.body, 'hi') self.assertEquals(req.headers['Content-Type'], 'text/plain') self.assertEquals(req.method, 'POST') body_file = StringIO('asdf') req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('')}, headers={'Content-Type': 'text/plain'}, body='hi', body_file=body_file) self.assert_(req.body_file is body_file) req = swift.common.swob.Request.blank( '/', environ={'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('')}, headers={'Content-Type': 'text/plain'}, body='hi', content_length=3) self.assertEquals(req.content_length, 3) self.assertEquals(len(req.body), 2) def test_blank_parsing(self): req = swift.common.swob.Request.blank('http://test.com/') self.assertEquals(req.environ['wsgi.url_scheme'], 'http') self.assertEquals(req.environ['SERVER_PORT'], '80') self.assertEquals(req.environ['SERVER_NAME'], 'test.com') req = swift.common.swob.Request.blank('https://test.com:456/') self.assertEquals(req.environ['wsgi.url_scheme'], 'https') self.assertEquals(req.environ['SERVER_PORT'], '456') req = swift.common.swob.Request.blank('test.com/') self.assertEquals(req.environ['wsgi.url_scheme'], 'http') self.assertEquals(req.environ['SERVER_PORT'], '80') self.assertEquals(req.environ['PATH_INFO'], 'test.com/') self.assertRaises(TypeError, swift.common.swob.Request.blank, 'ftp://test.com/') def test_params(self): req = swift.common.swob.Request.blank('/?a=b&c=d') self.assertEquals(req.params['a'], 'b') self.assertEquals(req.params['c'], 'd') def test_path(self): req = swift.common.swob.Request.blank('/hi?a=b&c=d') self.assertEquals(req.path, '/hi') req = swift.common.swob.Request.blank( '/', environ={'SCRIPT_NAME': '/hi', 'PATH_INFO': '/there'}) self.assertEquals(req.path, '/hi/there') def test_path_question_mark(self): req = swift.common.swob.Request.blank('/test%3Ffile') # This tests that .blank unquotes the path when setting PATH_INFO self.assertEquals(req.environ['PATH_INFO'], '/test?file') # This tests that .path requotes it self.assertEquals(req.path, '/test%3Ffile') def test_path_info_pop(self): req = swift.common.swob.Request.blank('/hi/there') self.assertEquals(req.path_info_pop(), 'hi') self.assertEquals(req.path_info, '/there') self.assertEquals(req.script_name, '/hi') def test_bad_path_info_pop(self): req = swift.common.swob.Request.blank('blahblah') self.assertEquals(req.path_info_pop(), None) def test_path_info_pop_last(self): req = swift.common.swob.Request.blank('/last') self.assertEquals(req.path_info_pop(), 'last') self.assertEquals(req.path_info, '') self.assertEquals(req.script_name, '/last') def test_path_info_pop_none(self): req = swift.common.swob.Request.blank('/') self.assertEquals(req.path_info_pop(), '') self.assertEquals(req.path_info, '') self.assertEquals(req.script_name, '/') def test_copy_get(self): req = swift.common.swob.Request.blank( '/hi/there', environ={'REQUEST_METHOD': 'POST'}) self.assertEquals(req.method, 'POST') req2 = req.copy_get() self.assertEquals(req2.method, 'GET') def test_get_response(self): def test_app(environ, start_response): start_response('200 OK', []) return ['hi'] req = swift.common.swob.Request.blank('/') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body, 'hi') def test_401_unauthorized(self): # No request environment resp = swift.common.swob.HTTPUnauthorized() self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) # Request environment req = swift.common.swob.Request.blank('/') resp = swift.common.swob.HTTPUnauthorized(request=req) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) def test_401_valid_account_path(self): def test_app(environ, start_response): start_response('401 Unauthorized', []) return ['hi'] # Request environment contains valid account in path req = swift.common.swob.Request.blank('/v1/account-name') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Swift realm="account-name"', resp.headers['Www-Authenticate']) # Request environment contains valid account/container in path req = swift.common.swob.Request.blank('/v1/account-name/c') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Swift realm="account-name"', resp.headers['Www-Authenticate']) def test_401_invalid_path(self): def test_app(environ, start_response): start_response('401 Unauthorized', []) return ['hi'] # Request environment contains bad path req = swift.common.swob.Request.blank('/random') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Swift realm="unknown"', resp.headers['Www-Authenticate']) def test_401_non_keystone_auth_path(self): def test_app(environ, start_response): start_response('401 Unauthorized', []) return ['no creds in request'] # Request to get token req = swift.common.swob.Request.blank('/v1.0/auth') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Swift realm="unknown"', resp.headers['Www-Authenticate']) # Other form of path req = swift.common.swob.Request.blank('/auth/v1.0') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Swift realm="unknown"', resp.headers['Www-Authenticate']) def test_401_www_authenticate_exists(self): def test_app(environ, start_response): start_response('401 Unauthorized', { 'Www-Authenticate': 'Me realm="whatever"'}) return ['no creds in request'] # Auth middleware sets own Www-Authenticate req = swift.common.swob.Request.blank('/auth/v1.0') resp = req.get_response(test_app) self.assertEquals(resp.status_int, 401) self.assert_('Www-Authenticate' in resp.headers) self.assertEquals('Me realm="whatever"', resp.headers['Www-Authenticate']) def test_not_401(self): # Other status codes should not have WWW-Authenticate in response def test_app(environ, start_response): start_response('200 OK', []) return ['hi'] req = swift.common.swob.Request.blank('/') resp = req.get_response(test_app) self.assert_('Www-Authenticate' not in resp.headers) def test_properties(self): req = swift.common.swob.Request.blank('/hi/there', body='hi') self.assertEquals(req.body, 'hi') self.assertEquals(req.content_length, 2) req.remote_addr = 'something' self.assertEquals(req.environ['REMOTE_ADDR'], 'something') req.body = 'whatever' self.assertEquals(req.content_length, 8) self.assertEquals(req.body, 'whatever') self.assertEquals(req.method, 'GET') req.range = 'bytes=1-7' self.assertEquals(req.range.ranges[0], (1, 7)) self.assert_('Range' in req.headers) req.range = None self.assert_('Range' not in req.headers) def test_datetime_properties(self): req = swift.common.swob.Request.blank('/hi/there', body='hi') req.if_unmodified_since = 0 self.assert_(isinstance(req.if_unmodified_since, datetime.datetime)) if_unmodified_since = req.if_unmodified_since req.if_unmodified_since = if_unmodified_since self.assertEquals(if_unmodified_since, req.if_unmodified_since) req.if_unmodified_since = 'something' self.assertEquals(req.headers['If-Unmodified-Since'], 'something') self.assertEquals(req.if_unmodified_since, None) self.assert_('If-Unmodified-Since' in req.headers) req.if_unmodified_since = None self.assert_('If-Unmodified-Since' not in req.headers) too_big_date_list = list(datetime.datetime.max.timetuple()) too_big_date_list[0] += 1 # bump up the year too_big_date = time.strftime( "%a, %d %b %Y %H:%M:%S UTC", time.struct_time(too_big_date_list)) req.if_unmodified_since = too_big_date self.assertEqual(req.if_unmodified_since, None) def test_bad_range(self): req = swift.common.swob.Request.blank('/hi/there', body='hi') req.range = 'bad range' self.assertEquals(req.range, None) def test_accept_header(self): req = swift.common.swob.Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/', 'HTTP_ACCEPT': 'application/json'}) self.assertEqual( req.accept.best_match(['application/json', 'text/plain']), 'application/json') self.assertEqual( req.accept.best_match(['text/plain', 'application/json']), 'application/json') def test_swift_entity_path(self): req = swift.common.swob.Request.blank('/v1/a/c/o') self.assertEqual(req.swift_entity_path, '/a/c/o') req = swift.common.swob.Request.blank('/v1/a/c') self.assertEqual(req.swift_entity_path, '/a/c') req = swift.common.swob.Request.blank('/v1/a') self.assertEqual(req.swift_entity_path, '/a') req = swift.common.swob.Request.blank('/v1') self.assertEqual(req.swift_entity_path, None) def test_path_qs(self): req = swift.common.swob.Request.blank('/hi/there?hello=equal&acl') self.assertEqual(req.path_qs, '/hi/there?hello=equal&acl') req = swift.common.swob.Request({'PATH_INFO': '/hi/there', 'QUERY_STRING': 'hello=equal&acl'}) self.assertEqual(req.path_qs, '/hi/there?hello=equal&acl') def test_url(self): req = swift.common.swob.Request.blank('/hi/there?hello=equal&acl') self.assertEqual(req.url, 'http://localhost/hi/there?hello=equal&acl') def test_wsgify(self): used_req = [] @swift.common.swob.wsgify def _wsgi_func(req): used_req.append(req) return swift.common.swob.Response('200 OK') req = swift.common.swob.Request.blank('/hi/there') resp = req.get_response(_wsgi_func) self.assertEqual(used_req[0].path, '/hi/there') self.assertEqual(resp.status_int, 200) def test_wsgify_raise(self): used_req = [] @swift.common.swob.wsgify def _wsgi_func(req): used_req.append(req) raise swift.common.swob.HTTPServerError() req = swift.common.swob.Request.blank('/hi/there') resp = req.get_response(_wsgi_func) self.assertEqual(used_req[0].path, '/hi/there') self.assertEqual(resp.status_int, 500) def test_split_path(self): """ Copied from swift.common.utils.split_path """ def _test_split_path(path, minsegs=1, maxsegs=None, rwl=False): req = swift.common.swob.Request.blank(path) return req.split_path(minsegs, maxsegs, rwl) self.assertRaises(ValueError, _test_split_path, '') self.assertRaises(ValueError, _test_split_path, '/') self.assertRaises(ValueError, _test_split_path, '//') self.assertEquals(_test_split_path('/a'), ['a']) self.assertRaises(ValueError, _test_split_path, '//a') self.assertEquals(_test_split_path('/a/'), ['a']) self.assertRaises(ValueError, _test_split_path, '/a/c') self.assertRaises(ValueError, _test_split_path, '//c') self.assertRaises(ValueError, _test_split_path, '/a/c/') self.assertRaises(ValueError, _test_split_path, '/a//') self.assertRaises(ValueError, _test_split_path, '/a', 2) self.assertRaises(ValueError, _test_split_path, '/a', 2, 3) self.assertRaises(ValueError, _test_split_path, '/a', 2, 3, True) self.assertEquals(_test_split_path('/a/c', 2), ['a', 'c']) self.assertEquals(_test_split_path('/a/c/o', 3), ['a', 'c', 'o']) self.assertRaises(ValueError, _test_split_path, '/a/c/o/r', 3, 3) self.assertEquals(_test_split_path('/a/c/o/r', 3, 3, True), ['a', 'c', 'o/r']) self.assertEquals(_test_split_path('/a/c', 2, 3, True), ['a', 'c', None]) self.assertRaises(ValueError, _test_split_path, '/a', 5, 4) self.assertEquals(_test_split_path('/a/c/', 2), ['a', 'c']) self.assertEquals(_test_split_path('/a/c/', 2, 3), ['a', 'c', '']) try: _test_split_path('o\nn e', 2) except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') try: _test_split_path('o\nn e', 2, 3, True) except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') def test_unicode_path(self): req = swift.common.swob.Request.blank(u'/\u2661') self.assertEquals(req.path, quote(u'/\u2661'.encode('utf-8'))) def test_unicode_query(self): req = swift.common.swob.Request.blank(u'/') req.query_string = u'x=\u2661' self.assertEquals(req.params['x'], u'\u2661'.encode('utf-8')) def test_url2(self): pi = '/hi/there' path = pi req = swift.common.swob.Request.blank(path) sche = 'http' exp_url = '%s://localhost%s' % (sche, pi) self.assertEqual(req.url, exp_url) qs = 'hello=equal&acl' path = '%s?%s' % (pi, qs) s, p = 'unit.test.example.com', '90' req = swift.common.swob.Request({'PATH_INFO': pi, 'QUERY_STRING': qs, 'SERVER_NAME': s, 'SERVER_PORT': p}) exp_url = '%s://%s:%s%s?%s' % (sche, s, p, pi, qs) self.assertEqual(req.url, exp_url) host = 'unit.test.example.com' req = swift.common.swob.Request({'PATH_INFO': pi, 'QUERY_STRING': qs, 'HTTP_HOST': host + ':80'}) exp_url = '%s://%s%s?%s' % (sche, host, pi, qs) self.assertEqual(req.url, exp_url) host = 'unit.test.example.com' sche = 'https' req = swift.common.swob.Request({'PATH_INFO': pi, 'QUERY_STRING': qs, 'HTTP_HOST': host + ':443', 'wsgi.url_scheme': sche}) exp_url = '%s://%s%s?%s' % (sche, host, pi, qs) self.assertEqual(req.url, exp_url) host = 'unit.test.example.com:81' req = swift.common.swob.Request({'PATH_INFO': pi, 'QUERY_STRING': qs, 'HTTP_HOST': host, 'wsgi.url_scheme': sche}) exp_url = '%s://%s%s?%s' % (sche, host, pi, qs) self.assertEqual(req.url, exp_url) def test_as_referer(self): pi = '/hi/there' qs = 'hello=equal&acl' sche = 'https' host = 'unit.test.example.com:81' req = swift.common.swob.Request({'REQUEST_METHOD': 'POST', 'PATH_INFO': pi, 'QUERY_STRING': qs, 'HTTP_HOST': host, 'wsgi.url_scheme': sche}) exp_url = '%s://%s%s?%s' % (sche, host, pi, qs) self.assertEqual(req.as_referer(), 'POST ' + exp_url) def test_message_length_just_content_length(self): req = swift.common.swob.Request.blank( u'/', environ={'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/'}) self.assertEquals(req.message_length(), None) req = swift.common.swob.Request.blank( u'/', environ={'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/'}, body='x' * 42) self.assertEquals(req.message_length(), 42) req.headers['Content-Length'] = 'abc' try: req.message_length() except ValueError as e: self.assertEquals(str(e), "Invalid Content-Length header value") else: self.fail("Expected a ValueError raised for 'abc'") def test_message_length_transfer_encoding(self): req = swift.common.swob.Request.blank( u'/', environ={'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/'}, headers={'transfer-encoding': 'chunked'}, body='x' * 42) self.assertEquals(req.message_length(), None) req.headers['Transfer-Encoding'] = 'gzip,chunked' try: req.message_length() except AttributeError as e: self.assertEquals(str(e), "Unsupported Transfer-Coding header" " value specified in Transfer-Encoding header") else: self.fail("Expected an AttributeError raised for 'gzip'") req.headers['Transfer-Encoding'] = 'gzip' try: req.message_length() except ValueError as e: self.assertEquals(str(e), "Invalid Transfer-Encoding header value") else: self.fail("Expected a ValueError raised for 'gzip'") req.headers['Transfer-Encoding'] = 'gzip,identity' try: req.message_length() except AttributeError as e: self.assertEquals(str(e), "Unsupported Transfer-Coding header" " value specified in Transfer-Encoding header") else: self.fail("Expected an AttributeError raised for 'gzip,identity'") class TestStatusMap(unittest.TestCase): def test_status_map(self): response_args = [] def start_response(status, headers): response_args.append(status) response_args.append(headers) resp_cls = swift.common.swob.status_map[404] resp = resp_cls() self.assertEquals(resp.status_int, 404) self.assertEquals(resp.title, 'Not Found') body = ''.join(resp({}, start_response)) self.assert_('The resource could not be found.' in body) self.assertEquals(response_args[0], '404 Not Found') headers = dict(response_args[1]) self.assertEquals(headers['Content-Type'], 'text/html; charset=UTF-8') self.assert_(int(headers['Content-Length']) > 0) class TestResponse(unittest.TestCase): def _get_response(self): def test_app(environ, start_response): start_response('200 OK', []) return ['hi'] req = swift.common.swob.Request.blank('/') return req.get_response(test_app) def test_properties(self): resp = self._get_response() resp.location = 'something' self.assertEquals(resp.location, 'something') self.assert_('Location' in resp.headers) resp.location = None self.assert_('Location' not in resp.headers) resp.content_type = 'text/plain' self.assert_('Content-Type' in resp.headers) resp.content_type = None self.assert_('Content-Type' not in resp.headers) def test_empty_body(self): resp = self._get_response() resp.body = '' self.assertEquals(resp.body, '') def test_unicode_body(self): resp = self._get_response() resp.body = u'\N{SNOWMAN}' self.assertEquals(resp.body, u'\N{SNOWMAN}'.encode('utf-8')) def test_call_reifies_request_if_necessary(self): """ The actual bug was a HEAD response coming out with a body because the Request object wasn't passed into the Response object's constructor. The Response object's __call__ method should be able to reify a Request object from the env it gets passed. """ def test_app(environ, start_response): start_response('200 OK', []) return ['hi'] req = swift.common.swob.Request.blank('/') req.method = 'HEAD' status, headers, app_iter = req.call_application(test_app) resp = swift.common.swob.Response(status=status, headers=dict(headers), app_iter=app_iter) output_iter = resp(req.environ, lambda *_: None) self.assertEquals(list(output_iter), ['']) def test_call_preserves_closeability(self): def test_app(environ, start_response): start_response('200 OK', []) yield "igloo" yield "shindig" yield "macadamia" yield "hullabaloo" req = swift.common.swob.Request.blank('/') req.method = 'GET' status, headers, app_iter = req.call_application(test_app) iterator = iter(app_iter) self.assertEqual('igloo', iterator.next()) self.assertEqual('shindig', iterator.next()) app_iter.close() self.assertRaises(StopIteration, iterator.next) def test_location_rewrite(self): def start_response(env, headers): pass req = swift.common.swob.Request.blank( '/', environ={'HTTP_HOST': 'somehost'}) resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://somehost/something') req = swift.common.swob.Request.blank( '/', environ={'HTTP_HOST': 'somehost:80'}) resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://somehost/something') req = swift.common.swob.Request.blank( '/', environ={'HTTP_HOST': 'somehost:443', 'wsgi.url_scheme': 'http'}) resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://somehost:443/something') req = swift.common.swob.Request.blank( '/', environ={'HTTP_HOST': 'somehost:443', 'wsgi.url_scheme': 'https'}) resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'https://somehost/something') def test_location_rewrite_no_host(self): def start_response(env, headers): pass req = swift.common.swob.Request.blank( '/', environ={'SERVER_NAME': 'local', 'SERVER_PORT': 80}) del req.environ['HTTP_HOST'] resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://local/something') req = swift.common.swob.Request.blank( '/', environ={'SERVER_NAME': 'local', 'SERVER_PORT': 81}) del req.environ['HTTP_HOST'] resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://local:81/something') def test_location_no_rewrite(self): def start_response(env, headers): pass req = swift.common.swob.Request.blank( '/', environ={'HTTP_HOST': 'somehost'}) resp = self._get_response() resp.location = 'http://www.google.com/' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, 'http://www.google.com/') def test_location_no_rewrite_when_told_not_to(self): def start_response(env, headers): pass req = swift.common.swob.Request.blank( '/', environ={'SERVER_NAME': 'local', 'SERVER_PORT': 81, 'swift.leave_relative_location': True}) del req.environ['HTTP_HOST'] resp = self._get_response() resp.location = '/something' # read response ''.join(resp(req.environ, start_response)) self.assertEquals(resp.location, '/something') def test_app_iter(self): def start_response(env, headers): pass resp = self._get_response() resp.app_iter = ['a', 'b', 'c'] body = ''.join(resp({}, start_response)) self.assertEquals(body, 'abc') def test_multi_ranges_wo_iter_ranges(self): def test_app(environ, start_response): start_response('200 OK', [('Content-Length', '10')]) return ['1234567890'] req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=0-9,10-19,20-29'}) resp = req.get_response(test_app) resp.conditional_response = True resp.content_length = 10 # read response ''.join(resp._response_iter(resp.app_iter, '')) self.assertEquals(resp.status, '200 OK') self.assertEqual(10, resp.content_length) def test_single_range_wo_iter_range(self): def test_app(environ, start_response): start_response('200 OK', [('Content-Length', '10')]) return ['1234567890'] req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=0-9'}) resp = req.get_response(test_app) resp.conditional_response = True resp.content_length = 10 # read response ''.join(resp._response_iter(resp.app_iter, '')) self.assertEquals(resp.status, '200 OK') self.assertEqual(10, resp.content_length) def test_multi_range_body(self): def test_app(environ, start_response): start_response('200 OK', [('Content-Length', '4')]) return ['abcd'] req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=0-9,10-19,20-29'}) resp = req.get_response(test_app) resp.conditional_response = True resp.content_length = 100 resp.content_type = 'text/plain' content = ''.join(resp._response_iter(None, ('0123456789112345678' '92123456789'))) self.assert_(re.match(('\r\n' '--[a-f0-9]{32}\r\n' 'Content-Type: text/plain\r\n' 'Content-Range: bytes ' '0-9/100\r\n\r\n0123456789\r\n' '--[a-f0-9]{32}\r\n' 'Content-Type: text/plain\r\n' 'Content-Range: bytes ' '10-19/100\r\n\r\n1123456789\r\n' '--[a-f0-9]{32}\r\n' 'Content-Type: text/plain\r\n' 'Content-Range: bytes ' '20-29/100\r\n\r\n2123456789\r\n' '--[a-f0-9]{32}--\r\n'), content)) def test_multi_response_iter(self): def test_app(environ, start_response): start_response('200 OK', [('Content-Length', '10'), ('Content-Type', 'application/xml')]) return ['0123456789'] app_iter_ranges_args = [] class App_iter(object): def app_iter_ranges(self, ranges, content_type, boundary, size): app_iter_ranges_args.append((ranges, content_type, boundary, size)) for i in xrange(3): yield str(i) + 'fun' yield boundary def __iter__(self): for i in xrange(3): yield str(i) + 'fun' req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=1-5,8-11'}) resp = req.get_response(test_app) resp.conditional_response = True resp.content_length = 12 content = ''.join(resp._response_iter(App_iter(), '')) boundary = content[-32:] self.assertEqual(content[:-32], '0fun1fun2fun') self.assertEqual(app_iter_ranges_args, [([(1, 6), (8, 12)], 'application/xml', boundary, 12)]) def test_range_body(self): def test_app(environ, start_response): start_response('200 OK', [('Content-Length', '10')]) return ['1234567890'] def start_response(env, headers): pass req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=1-3'}) resp = swift.common.swob.Response( body='1234567890', request=req, conditional_response=True) body = ''.join(resp([], start_response)) self.assertEquals(body, '234') self.assertEquals(resp.content_range, 'bytes 1-3/10') self.assertEquals(resp.status, '206 Partial Content') # syntactically valid, but does not make sense, so returning 416 # in next couple of cases. req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=-0'}) resp = req.get_response(test_app) resp.conditional_response = True body = ''.join(resp([], start_response)) self.assertEquals(body, '') self.assertEquals(resp.content_length, 0) self.assertEquals(resp.status, '416 Requested Range Not Satisfiable') resp = swift.common.swob.Response( body='1234567890', request=req, conditional_response=True) body = ''.join(resp([], start_response)) self.assertEquals(body, '') self.assertEquals(resp.content_length, 0) self.assertEquals(resp.status, '416 Requested Range Not Satisfiable') # Syntactically-invalid Range headers "MUST" be ignored req = swift.common.swob.Request.blank( '/', headers={'Range': 'bytes=3-2'}) resp = req.get_response(test_app) resp.conditional_response = True body = ''.join(resp([], start_response)) self.assertEquals(body, '1234567890') self.assertEquals(resp.status, '200 OK') resp = swift.common.swob.Response( body='1234567890', request=req, conditional_response=True) body = ''.join(resp([], start_response)) self.assertEquals(body, '1234567890') self.assertEquals(resp.status, '200 OK') def test_content_type(self): resp = self._get_response() resp.content_type = 'text/plain; charset=utf8' self.assertEquals(resp.content_type, 'text/plain') def test_charset(self): resp = self._get_response() resp.content_type = 'text/plain; charset=utf8' self.assertEquals(resp.charset, 'utf8') resp.charset = 'utf16' self.assertEquals(resp.charset, 'utf16') def test_charset_content_type(self): resp = swift.common.swob.Response( content_type='text/plain', charset='utf-8') self.assertEquals(resp.charset, 'utf-8') resp = swift.common.swob.Response( charset='utf-8', content_type='text/plain') self.assertEquals(resp.charset, 'utf-8') def test_etag(self): resp = self._get_response() resp.etag = 'hi' self.assertEquals(resp.headers['Etag'], '"hi"') self.assertEquals(resp.etag, 'hi') self.assert_('etag' in resp.headers) resp.etag = None self.assert_('etag' not in resp.headers) def test_host_url_default(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'http' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '1234' del env['HTTP_HOST'] self.assertEquals(resp.host_url, 'http://bob:1234') def test_host_url_default_port_squelched(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'http' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '80' del env['HTTP_HOST'] self.assertEquals(resp.host_url, 'http://bob') def test_host_url_https(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'https' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '1234' del env['HTTP_HOST'] self.assertEquals(resp.host_url, 'https://bob:1234') def test_host_url_https_port_squelched(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'https' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '443' del env['HTTP_HOST'] self.assertEquals(resp.host_url, 'https://bob') def test_host_url_host_override(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'http' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '1234' env['HTTP_HOST'] = 'someother' self.assertEquals(resp.host_url, 'http://someother') def test_host_url_host_port_override(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'http' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '1234' env['HTTP_HOST'] = 'someother:5678' self.assertEquals(resp.host_url, 'http://someother:5678') def test_host_url_host_https(self): resp = self._get_response() env = resp.environ env['wsgi.url_scheme'] = 'https' env['SERVER_NAME'] = 'bob' env['SERVER_PORT'] = '1234' env['HTTP_HOST'] = 'someother:5678' self.assertEquals(resp.host_url, 'https://someother:5678') def test_507(self): resp = swift.common.swob.HTTPInsufficientStorage() content = ''.join(resp._response_iter(resp.app_iter, resp._body)) self.assertEquals( content, '

Insufficient Storage

There was not enough space ' 'to save the resource. Drive: unknown

') resp = swift.common.swob.HTTPInsufficientStorage(drive='sda1') content = ''.join(resp._response_iter(resp.app_iter, resp._body)) self.assertEquals( content, '

Insufficient Storage

There was not enough space ' 'to save the resource. Drive: sda1

') class TestUTC(unittest.TestCase): def test_tzname(self): self.assertEquals(swift.common.swob.UTC.tzname(None), 'UTC') class TestConditionalIfNoneMatch(unittest.TestCase): def fake_app(self, environ, start_response): start_response('200 OK', [('Etag', 'the-etag')]) return ['hi'] def fake_start_response(*a, **kw): pass def test_simple_match(self): # etag matches --> 304 req = swift.common.swob.Request.blank( '/', headers={'If-None-Match': 'the-etag'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 304) self.assertEquals(body, '') def test_quoted_simple_match(self): # double quotes don't matter req = swift.common.swob.Request.blank( '/', headers={'If-None-Match': '"the-etag"'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 304) self.assertEquals(body, '') def test_list_match(self): # it works with lists of etags to match req = swift.common.swob.Request.blank( '/', headers={'If-None-Match': '"bert", "the-etag", "ernie"'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 304) self.assertEquals(body, '') def test_list_no_match(self): # no matches --> whatever the original status was req = swift.common.swob.Request.blank( '/', headers={'If-None-Match': '"bert", "ernie"'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 200) self.assertEquals(body, 'hi') def test_match_star(self): # "*" means match anything; see RFC 2616 section 14.24 req = swift.common.swob.Request.blank( '/', headers={'If-None-Match': '*'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 304) self.assertEquals(body, '') class TestConditionalIfMatch(unittest.TestCase): def fake_app(self, environ, start_response): start_response('200 OK', [('Etag', 'the-etag')]) return ['hi'] def fake_start_response(*a, **kw): pass def test_simple_match(self): # if etag matches, proceed as normal req = swift.common.swob.Request.blank( '/', headers={'If-Match': 'the-etag'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 200) self.assertEquals(body, 'hi') def test_quoted_simple_match(self): # double quotes or not, doesn't matter req = swift.common.swob.Request.blank( '/', headers={'If-Match': '"the-etag"'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 200) self.assertEquals(body, 'hi') def test_no_match(self): # no match --> 412 req = swift.common.swob.Request.blank( '/', headers={'If-Match': 'not-the-etag'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 412) self.assertEquals(body, '') def test_match_star(self): # "*" means match anything; see RFC 2616 section 14.24 req = swift.common.swob.Request.blank( '/', headers={'If-Match': '*'}) resp = req.get_response(self.fake_app) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 200) self.assertEquals(body, 'hi') def test_match_star_on_404(self): def fake_app_404(environ, start_response): start_response('404 Not Found', []) return ['hi'] req = swift.common.swob.Request.blank( '/', headers={'If-Match': '*'}) resp = req.get_response(fake_app_404) resp.conditional_response = True body = ''.join(resp(req.environ, self.fake_start_response)) self.assertEquals(resp.status_int, 412) self.assertEquals(body, '') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_db.py0000664000175400017540000007002112323703611021311 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.common.db""" import os import unittest from tempfile import mkdtemp from shutil import rmtree, copy from uuid import uuid4 import simplejson import sqlite3 from mock import patch, MagicMock from eventlet.timeout import Timeout import swift.common.db from swift.common.db import chexor, dict_factory, get_db_connection, \ DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \ GreenDBConnection from swift.common.utils import normalize_timestamp, mkdirs from swift.common.exceptions import LockTimeout class TestDatabaseConnectionError(unittest.TestCase): def test_str(self): err = \ DatabaseConnectionError(':memory:', 'No valid database connection') self.assert_(':memory:' in str(err)) self.assert_('No valid database connection' in str(err)) err = DatabaseConnectionError(':memory:', 'No valid database connection', timeout=1357) self.assert_(':memory:' in str(err)) self.assert_('No valid database connection' in str(err)) self.assert_('1357' in str(err)) class TestDictFactory(unittest.TestCase): def test_normal_case(self): conn = sqlite3.connect(':memory:') conn.execute('CREATE TABLE test (one TEXT, two INTEGER)') conn.execute('INSERT INTO test (one, two) VALUES ("abc", 123)') conn.execute('INSERT INTO test (one, two) VALUES ("def", 456)') conn.commit() curs = conn.execute('SELECT one, two FROM test') self.assertEquals(dict_factory(curs, curs.next()), {'one': 'abc', 'two': 123}) self.assertEquals(dict_factory(curs, curs.next()), {'one': 'def', 'two': 456}) class TestChexor(unittest.TestCase): def test_normal_case(self): self.assertEquals( chexor('d41d8cd98f00b204e9800998ecf8427e', 'new name', normalize_timestamp(1)), '4f2ea31ac14d4273fe32ba08062b21de') def test_invalid_old_hash(self): self.assertRaises(ValueError, chexor, 'oldhash', 'name', normalize_timestamp(1)) def test_no_name(self): self.assertRaises(Exception, chexor, 'd41d8cd98f00b204e9800998ecf8427e', None, normalize_timestamp(1)) class TestGreenDBConnection(unittest.TestCase): def test_execute_when_locked(self): # This test is dependent on the code under test calling execute and # commit as sqlite3.Cursor.execute in a subclass. class InterceptCursor(sqlite3.Cursor): pass db_error = sqlite3.OperationalError('database is locked') InterceptCursor.execute = MagicMock(side_effect=db_error) with patch('sqlite3.Cursor', new=InterceptCursor): conn = sqlite3.connect(':memory:', check_same_thread=False, factory=GreenDBConnection, timeout=0.1) self.assertRaises(Timeout, conn.execute, 'select 1') self.assertTrue(InterceptCursor.execute.called) self.assertEqual(InterceptCursor.execute.call_args_list, list((InterceptCursor.execute.call_args,) * InterceptCursor.execute.call_count)) def text_commit_when_locked(self): # This test is dependent on the code under test calling commit and # commit as sqlite3.Connection.commit in a subclass. class InterceptConnection(sqlite3.Connection): pass db_error = sqlite3.OperationalError('database is locked') InterceptConnection.commit = MagicMock(side_effect=db_error) with patch('sqlite3.Connection', new=InterceptConnection): conn = sqlite3.connect(':memory:', check_same_thread=False, factory=GreenDBConnection, timeout=0.1) self.assertRaises(Timeout, conn.commit) self.assertTrue(InterceptConnection.commit.called) self.assertEqual(InterceptConnection.commit.call_args_list, list((InterceptConnection.commit.call_args,) * InterceptConnection.commit.call_count)) class TestGetDBConnection(unittest.TestCase): def test_normal_case(self): conn = get_db_connection(':memory:') self.assert_(hasattr(conn, 'execute')) def test_invalid_path(self): self.assertRaises(DatabaseConnectionError, get_db_connection, 'invalid database path / name') def test_locked_db(self): # This test is dependent on the code under test calling execute and # commit as sqlite3.Cursor.execute in a subclass. class InterceptCursor(sqlite3.Cursor): pass db_error = sqlite3.OperationalError('database is locked') mock_db_cmd = MagicMock(side_effect=db_error) InterceptCursor.execute = mock_db_cmd with patch('sqlite3.Cursor', new=InterceptCursor): self.assertRaises(Timeout, get_db_connection, ':memory:', timeout=0.1) self.assertTrue(mock_db_cmd.called) self.assertEqual(mock_db_cmd.call_args_list, list((mock_db_cmd.call_args,) * mock_db_cmd.call_count)) class TestDatabaseBroker(unittest.TestCase): def setUp(self): self.testdir = mkdtemp() def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_DB_PREALLOCATION_setting(self): u = uuid4().hex b = DatabaseBroker(u) swift.common.db.DB_PREALLOCATION = False b._preallocate() swift.common.db.DB_PREALLOCATION = True self.assertRaises(OSError, b._preallocate) def test_memory_db_init(self): broker = DatabaseBroker(':memory:') self.assertEqual(broker.db_file, ':memory:') self.assertRaises(AttributeError, broker.initialize, normalize_timestamp('0')) def test_disk_db_init(self): db_file = os.path.join(self.testdir, '1.db') broker = DatabaseBroker(db_file) self.assertEqual(broker.db_file, db_file) self.assert_(broker.conn is None) def test_disk_preallocate(self): test_size = [-1] def fallocate_stub(fd, size): test_size[0] = size with patch('swift.common.db.fallocate', fallocate_stub): db_file = os.path.join(self.testdir, 'pre.db') # Write 1 byte and hope that the fs will allocate less than 1 MB. f = open(db_file, "w") f.write('@') f.close() b = DatabaseBroker(db_file) b._preallocate() # We only wrote 1 byte, so we should end with the 1st step or 1 MB. self.assertEquals(test_size[0], 1024 * 1024) def test_initialize(self): self.assertRaises(AttributeError, DatabaseBroker(':memory:').initialize, normalize_timestamp('1')) stub_dict = {} def stub(*args, **kwargs): for key in stub_dict.keys(): del stub_dict[key] stub_dict['args'] = args for key, value in kwargs.items(): stub_dict[key] = value broker = DatabaseBroker(':memory:') broker._initialize = stub broker.initialize(normalize_timestamp('1')) self.assert_(hasattr(stub_dict['args'][0], 'execute')) self.assertEquals(stub_dict['args'][1], '0000000001.00000') with broker.get() as conn: conn.execute('SELECT * FROM outgoing_sync') conn.execute('SELECT * FROM incoming_sync') broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker._initialize = stub broker.initialize(normalize_timestamp('1')) self.assert_(hasattr(stub_dict['args'][0], 'execute')) self.assertEquals(stub_dict['args'][1], '0000000001.00000') with broker.get() as conn: conn.execute('SELECT * FROM outgoing_sync') conn.execute('SELECT * FROM incoming_sync') broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker._initialize = stub self.assertRaises(DatabaseAlreadyExists, broker.initialize, normalize_timestamp('1')) def test_delete_db(self): def init_stub(conn, put_timestamp): conn.execute('CREATE TABLE test (one TEXT)') conn.execute('CREATE TABLE test_stat (id TEXT)') conn.execute('INSERT INTO test_stat (id) VALUES (?)', (str(uuid4),)) conn.execute('INSERT INTO test (one) VALUES ("1")') conn.commit() stub_called = [False] def delete_stub(*a, **kw): stub_called[0] = True broker = DatabaseBroker(':memory:') broker.db_type = 'test' broker._initialize = init_stub # Initializes a good broker for us broker.initialize(normalize_timestamp('1')) self.assert_(broker.conn is not None) broker._delete_db = delete_stub stub_called[0] = False broker.delete_db('2') self.assert_(stub_called[0]) broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) broker.db_type = 'test' broker._initialize = init_stub broker.initialize(normalize_timestamp('1')) broker._delete_db = delete_stub stub_called[0] = False broker.delete_db('2') self.assert_(stub_called[0]) # ensure that metadata was cleared m2 = broker.metadata self.assert_(not any(v[0] for v in m2.itervalues())) self.assert_(all(v[1] == normalize_timestamp('2') for v in m2.itervalues())) def test_get(self): broker = DatabaseBroker(':memory:') got_exc = False try: with broker.get() as conn: conn.execute('SELECT 1') except Exception: got_exc = True broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) got_exc = False try: with broker.get() as conn: conn.execute('SELECT 1') except Exception: got_exc = True self.assert_(got_exc) def stub(*args, **kwargs): pass broker._initialize = stub broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('CREATE TABLE test (one TEXT)') try: with broker.get() as conn: conn.execute('INSERT INTO test (one) VALUES ("1")') raise Exception('test') conn.commit() except Exception: pass broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) with broker.get() as conn: self.assertEquals( [r[0] for r in conn.execute('SELECT * FROM test')], []) with broker.get() as conn: conn.execute('INSERT INTO test (one) VALUES ("1")') conn.commit() broker = DatabaseBroker(os.path.join(self.testdir, '1.db')) with broker.get() as conn: self.assertEquals( [r[0] for r in conn.execute('SELECT * FROM test')], ['1']) dbpath = os.path.join(self.testdir, 'dev', 'dbs', 'par', 'pre', 'db') mkdirs(dbpath) qpath = os.path.join(self.testdir, 'dev', 'quarantined', 'tests', 'db') with patch('swift.common.db.renamer', lambda a, b: b): # Test malformed database copy(os.path.join(os.path.dirname(__file__), 'malformed_example.db'), os.path.join(dbpath, '1.db')) broker = DatabaseBroker(os.path.join(dbpath, '1.db')) broker.db_type = 'test' exc = None try: with broker.get() as conn: conn.execute('SELECT * FROM test') except Exception as err: exc = err self.assertEquals( str(exc), 'Quarantined %s to %s due to malformed database' % (dbpath, qpath)) # Test corrupted database copy(os.path.join(os.path.dirname(__file__), 'corrupted_example.db'), os.path.join(dbpath, '1.db')) broker = DatabaseBroker(os.path.join(dbpath, '1.db')) broker.db_type = 'test' exc = None try: with broker.get() as conn: conn.execute('SELECT * FROM test') except Exception as err: exc = err self.assertEquals( str(exc), 'Quarantined %s to %s due to corrupted database' % (dbpath, qpath)) def test_lock(self): broker = DatabaseBroker(os.path.join(self.testdir, '1.db'), timeout=.1) got_exc = False try: with broker.lock(): pass except Exception: got_exc = True self.assert_(got_exc) def stub(*args, **kwargs): pass broker._initialize = stub broker.initialize(normalize_timestamp('1')) with broker.lock(): pass with broker.lock(): pass broker2 = DatabaseBroker(os.path.join(self.testdir, '1.db'), timeout=.1) broker2._initialize = stub with broker.lock(): got_exc = False try: with broker2.lock(): pass except LockTimeout: got_exc = True self.assert_(got_exc) try: with broker.lock(): raise Exception('test') except Exception: pass with broker.lock(): pass def test_newid(self): broker = DatabaseBroker(':memory:') broker.db_type = 'test' broker.db_contains_type = 'test' uuid1 = str(uuid4()) def _initialize(conn, timestamp): conn.execute('CREATE TABLE test (one TEXT)') conn.execute('CREATE TABLE test_stat (id TEXT)') conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,)) conn.commit() broker._initialize = _initialize broker.initialize(normalize_timestamp('1')) uuid2 = str(uuid4()) broker.newid(uuid2) with broker.get() as conn: uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] self.assertEquals(len(uuids), 1) self.assertNotEquals(uuids[0], uuid1) uuid1 = uuids[0] points = [(r[0], r[1]) for r in conn.execute( 'SELECT sync_point, ' 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid2,))] self.assertEquals(len(points), 1) self.assertEquals(points[0][0], -1) self.assertEquals(points[0][1], uuid2) conn.execute('INSERT INTO test (one) VALUES ("1")') conn.commit() uuid3 = str(uuid4()) broker.newid(uuid3) with broker.get() as conn: uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] self.assertEquals(len(uuids), 1) self.assertNotEquals(uuids[0], uuid1) uuid1 = uuids[0] points = [(r[0], r[1]) for r in conn.execute( 'SELECT sync_point, ' 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid3,))] self.assertEquals(len(points), 1) self.assertEquals(points[0][1], uuid3) broker.newid(uuid2) with broker.get() as conn: uuids = [r[0] for r in conn.execute('SELECT * FROM test_stat')] self.assertEquals(len(uuids), 1) self.assertNotEquals(uuids[0], uuid1) points = [(r[0], r[1]) for r in conn.execute( 'SELECT sync_point, ' 'remote_id FROM incoming_sync WHERE remote_id = ?', (uuid2,))] self.assertEquals(len(points), 1) self.assertEquals(points[0][1], uuid2) def test_get_items_since(self): broker = DatabaseBroker(':memory:') broker.db_type = 'test' broker.db_contains_type = 'test' def _initialize(conn, timestamp): conn.execute('CREATE TABLE test (one TEXT)') conn.execute('INSERT INTO test (one) VALUES ("1")') conn.execute('INSERT INTO test (one) VALUES ("2")') conn.execute('INSERT INTO test (one) VALUES ("3")') conn.commit() broker._initialize = _initialize broker.initialize(normalize_timestamp('1')) self.assertEquals(broker.get_items_since(-1, 10), [{'one': '1'}, {'one': '2'}, {'one': '3'}]) self.assertEquals(broker.get_items_since(-1, 2), [{'one': '1'}, {'one': '2'}]) self.assertEquals(broker.get_items_since(1, 2), [{'one': '2'}, {'one': '3'}]) self.assertEquals(broker.get_items_since(3, 2), []) self.assertEquals(broker.get_items_since(999, 2), []) def test_get_sync(self): broker = DatabaseBroker(':memory:') broker.db_type = 'test' broker.db_contains_type = 'test' uuid1 = str(uuid4()) def _initialize(conn, timestamp): conn.execute('CREATE TABLE test (one TEXT)') conn.execute('CREATE TABLE test_stat (id TEXT)') conn.execute('INSERT INTO test_stat (id) VALUES (?)', (uuid1,)) conn.execute('INSERT INTO test (one) VALUES ("1")') conn.commit() pass broker._initialize = _initialize broker.initialize(normalize_timestamp('1')) uuid2 = str(uuid4()) self.assertEquals(broker.get_sync(uuid2), -1) broker.newid(uuid2) self.assertEquals(broker.get_sync(uuid2), 1) uuid3 = str(uuid4()) self.assertEquals(broker.get_sync(uuid3), -1) with broker.get() as conn: conn.execute('INSERT INTO test (one) VALUES ("2")') conn.commit() broker.newid(uuid3) self.assertEquals(broker.get_sync(uuid2), 1) self.assertEquals(broker.get_sync(uuid3), 2) self.assertEquals(broker.get_sync(uuid2, incoming=False), -1) self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) broker.merge_syncs([{'sync_point': 1, 'remote_id': uuid2}], incoming=False) self.assertEquals(broker.get_sync(uuid2), 1) self.assertEquals(broker.get_sync(uuid3), 2) self.assertEquals(broker.get_sync(uuid2, incoming=False), 1) self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) broker.merge_syncs([{'sync_point': 2, 'remote_id': uuid3}], incoming=False) self.assertEquals(broker.get_sync(uuid2, incoming=False), 1) self.assertEquals(broker.get_sync(uuid3, incoming=False), 2) def test_merge_syncs(self): broker = DatabaseBroker(':memory:') def stub(*args, **kwargs): pass broker._initialize = stub broker.initialize(normalize_timestamp('1')) uuid2 = str(uuid4()) broker.merge_syncs([{'sync_point': 1, 'remote_id': uuid2}]) self.assertEquals(broker.get_sync(uuid2), 1) uuid3 = str(uuid4()) broker.merge_syncs([{'sync_point': 2, 'remote_id': uuid3}]) self.assertEquals(broker.get_sync(uuid2), 1) self.assertEquals(broker.get_sync(uuid3), 2) self.assertEquals(broker.get_sync(uuid2, incoming=False), -1) self.assertEquals(broker.get_sync(uuid3, incoming=False), -1) broker.merge_syncs([{'sync_point': 3, 'remote_id': uuid2}, {'sync_point': 4, 'remote_id': uuid3}], incoming=False) self.assertEquals(broker.get_sync(uuid2, incoming=False), 3) self.assertEquals(broker.get_sync(uuid3, incoming=False), 4) self.assertEquals(broker.get_sync(uuid2), 1) self.assertEquals(broker.get_sync(uuid3), 2) broker.merge_syncs([{'sync_point': 5, 'remote_id': uuid2}]) self.assertEquals(broker.get_sync(uuid2), 5) def test_get_replication_info(self): self.get_replication_info_tester(metadata=False) def test_get_replication_info_with_metadata(self): self.get_replication_info_tester(metadata=True) def get_replication_info_tester(self, metadata=False): broker = DatabaseBroker(':memory:', account='a') broker.db_type = 'test' broker.db_contains_type = 'test' broker_creation = normalize_timestamp(1) broker_uuid = str(uuid4()) broker_metadata = metadata and simplejson.dumps( {'Test': ('Value', normalize_timestamp(1))}) or '' def _initialize(conn, put_timestamp): if put_timestamp is None: put_timestamp = normalize_timestamp(0) conn.executescript(''' CREATE TABLE test ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, created_at TEXT ); CREATE TRIGGER test_insert AFTER INSERT ON test BEGIN UPDATE test_stat SET test_count = test_count + 1, hash = chexor(hash, new.name, new.created_at); END; CREATE TRIGGER test_update BEFORE UPDATE ON test BEGIN SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); END; CREATE TRIGGER test_delete AFTER DELETE ON test BEGIN UPDATE test_stat SET test_count = test_count - 1, hash = chexor(hash, old.name, old.created_at); END; CREATE TABLE test_stat ( account TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', test_count INTEGER, hash TEXT default '00000000000000000000000000000000', id TEXT %s ); INSERT INTO test_stat (test_count) VALUES (0); ''' % (metadata and ", metadata TEXT DEFAULT ''" or "")) conn.execute(''' UPDATE test_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ? ''', (broker.account, broker_creation, broker_uuid, put_timestamp)) if metadata: conn.execute('UPDATE test_stat SET metadata = ?', (broker_metadata,)) conn.commit() broker._initialize = _initialize put_timestamp = normalize_timestamp(2) broker.initialize(put_timestamp) info = broker.get_replication_info() self.assertEquals(info, { 'count': 0, 'hash': '00000000000000000000000000000000', 'created_at': broker_creation, 'put_timestamp': put_timestamp, 'delete_timestamp': '0', 'max_row': -1, 'id': broker_uuid, 'metadata': broker_metadata}) insert_timestamp = normalize_timestamp(3) with broker.get() as conn: conn.execute(''' INSERT INTO test (name, created_at) VALUES ('test', ?) ''', (insert_timestamp,)) conn.commit() info = broker.get_replication_info() self.assertEquals(info, { 'count': 1, 'hash': 'bdc4c93f574b0d8c2911a27ce9dd38ba', 'created_at': broker_creation, 'put_timestamp': put_timestamp, 'delete_timestamp': '0', 'max_row': 1, 'id': broker_uuid, 'metadata': broker_metadata}) with broker.get() as conn: conn.execute('DELETE FROM test') conn.commit() info = broker.get_replication_info() self.assertEquals(info, { 'count': 0, 'hash': '00000000000000000000000000000000', 'created_at': broker_creation, 'put_timestamp': put_timestamp, 'delete_timestamp': '0', 'max_row': 1, 'id': broker_uuid, 'metadata': broker_metadata}) return broker def test_metadata(self): def reclaim(broker, timestamp): with broker.get() as conn: broker._reclaim(conn, timestamp) conn.commit() # Initializes a good broker for us broker = self.get_replication_info_tester(metadata=True) # Add our first item first_timestamp = normalize_timestamp(1) first_value = '1' broker.update_metadata({'First': [first_value, first_timestamp]}) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) # Add our second item second_timestamp = normalize_timestamp(2) second_value = '2' broker.update_metadata({'Second': [second_value, second_timestamp]}) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' in broker.metadata) self.assertEquals(broker.metadata['Second'], [second_value, second_timestamp]) # Update our first item first_timestamp = normalize_timestamp(3) first_value = '1b' broker.update_metadata({'First': [first_value, first_timestamp]}) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' in broker.metadata) self.assertEquals(broker.metadata['Second'], [second_value, second_timestamp]) # Delete our second item (by setting to empty string) second_timestamp = normalize_timestamp(4) second_value = '' broker.update_metadata({'Second': [second_value, second_timestamp]}) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' in broker.metadata) self.assertEquals(broker.metadata['Second'], [second_value, second_timestamp]) # Reclaim at point before second item was deleted reclaim(broker, normalize_timestamp(3)) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' in broker.metadata) self.assertEquals(broker.metadata['Second'], [second_value, second_timestamp]) # Reclaim at point second item was deleted reclaim(broker, normalize_timestamp(4)) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' in broker.metadata) self.assertEquals(broker.metadata['Second'], [second_value, second_timestamp]) # Reclaim after point second item was deleted reclaim(broker, normalize_timestamp(5)) self.assert_('First' in broker.metadata) self.assertEquals(broker.metadata['First'], [first_value, first_timestamp]) self.assert_('Second' not in broker.metadata) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_constraints.py0000664000175400017540000003551612323703614023310 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import mock import tempfile from test import safe_repr from test.unit import MockTrue from swift.common.swob import HTTPBadRequest, Request, HTTPException from swift.common.http import HTTP_REQUEST_ENTITY_TOO_LARGE, \ HTTP_BAD_REQUEST, HTTP_LENGTH_REQUIRED from swift.common import constraints class TestConstraints(unittest.TestCase): def assertIn(self, member, container, msg=None): """Copied from 2.7""" if member not in container: standardMsg = '%s not found in %s' % (safe_repr(member), safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) def test_check_metadata_empty(self): headers = {} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) def test_check_metadata_good(self): headers = {'X-Object-Meta-Name': 'Value'} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) def test_check_metadata_empty_name(self): headers = {'X-Object-Meta-': 'Value'} self.assert_(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), HTTPBadRequest) def test_check_metadata_name_length(self): name = 'a' * constraints.MAX_META_NAME_LENGTH headers = {'X-Object-Meta-%s' % name: 'v'} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) name = 'a' * (constraints.MAX_META_NAME_LENGTH + 1) headers = {'X-Object-Meta-%s' % name: 'v'} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object').status_int, HTTP_BAD_REQUEST) self.assertIn( ('X-Object-Meta-%s' % name).lower(), constraints.check_metadata(Request.blank( '/', headers=headers), 'object').body.lower()) def test_check_metadata_value_length(self): value = 'a' * constraints.MAX_META_VALUE_LENGTH headers = {'X-Object-Meta-Name': value} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) value = 'a' * (constraints.MAX_META_VALUE_LENGTH + 1) headers = {'X-Object-Meta-Name': value} self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object').status_int, HTTP_BAD_REQUEST) self.assertIn( 'x-object-meta-name', constraints.check_metadata(Request.blank( '/', headers=headers), 'object').body.lower()) self.assertIn( str(constraints.MAX_META_VALUE_LENGTH), constraints.check_metadata(Request.blank( '/', headers=headers), 'object').body) def test_check_metadata_count(self): headers = {} for x in xrange(constraints.MAX_META_COUNT): headers['X-Object-Meta-%d' % x] = 'v' self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) headers['X-Object-Meta-Too-Many'] = 'v' self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object').status_int, HTTP_BAD_REQUEST) def test_check_metadata_size(self): headers = {} size = 0 chunk = constraints.MAX_META_NAME_LENGTH + \ constraints.MAX_META_VALUE_LENGTH x = 0 while size + chunk < constraints.MAX_META_OVERALL_SIZE: headers['X-Object-Meta-%04d%s' % (x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \ 'v' * constraints.MAX_META_VALUE_LENGTH size += chunk x += 1 self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object'), None) # add two more headers in case adding just one falls exactly on the # limit (eg one header adds 1024 and the limit is 2048) headers['X-Object-Meta-%04d%s' % (x, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \ 'v' * constraints.MAX_META_VALUE_LENGTH headers['X-Object-Meta-%04d%s' % (x + 1, 'a' * (constraints.MAX_META_NAME_LENGTH - 4))] = \ 'v' * constraints.MAX_META_VALUE_LENGTH self.assertEquals(constraints.check_metadata(Request.blank( '/', headers=headers), 'object').status_int, HTTP_BAD_REQUEST) def test_check_object_creation_content_length(self): headers = {'Content-Length': str(constraints.MAX_FILE_SIZE), 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name'), None) headers = {'Content-Length': str(constraints.MAX_FILE_SIZE + 1), 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation( Request.blank('/', headers=headers), 'object_name').status_int, HTTP_REQUEST_ENTITY_TOO_LARGE) headers = {'Transfer-Encoding': 'chunked', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name'), None) headers = {'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation( Request.blank('/', headers=headers), 'object_name').status_int, HTTP_LENGTH_REQUIRED) def test_check_object_creation_name_length(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': 'text/plain'} name = 'o' * constraints.MAX_OBJECT_NAME_LENGTH self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), name), None) name = 'o' * (constraints.MAX_OBJECT_NAME_LENGTH + 1) self.assertEquals(constraints.check_object_creation( Request.blank('/', headers=headers), name).status_int, HTTP_BAD_REQUEST) def test_check_object_creation_content_type(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': 'text/plain'} self.assertEquals(constraints.check_object_creation(Request.blank( '/', headers=headers), 'object_name'), None) headers = {'Transfer-Encoding': 'chunked'} self.assertEquals(constraints.check_object_creation( Request.blank('/', headers=headers), 'object_name').status_int, HTTP_BAD_REQUEST) def test_check_object_creation_bad_content_type(self): headers = {'Transfer-Encoding': 'chunked', 'Content-Type': '\xff\xff'} resp = constraints.check_object_creation( Request.blank('/', headers=headers), 'object_name') self.assertEquals(resp.status_int, HTTP_BAD_REQUEST) self.assert_('Content-Type' in resp.body) def test_check_mount(self): self.assertFalse(constraints.check_mount('', '')) with mock.patch("swift.common.constraints.ismount", MockTrue()): self.assertTrue(constraints.check_mount('/srv', '1')) self.assertTrue(constraints.check_mount('/srv', 'foo-bar')) self.assertTrue(constraints.check_mount( '/srv', '003ed03c-242a-4b2f-bee9-395f801d1699')) self.assertFalse(constraints.check_mount('/srv', 'foo bar')) self.assertFalse(constraints.check_mount('/srv', 'foo/bar')) self.assertFalse(constraints.check_mount('/srv', 'foo?bar')) def test_check_float(self): self.assertFalse(constraints.check_float('')) self.assertTrue(constraints.check_float('0')) def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') invalid_utf8_str = unicode_sample.encode('utf-8')[::-1] unicode_with_null = u'abc\u0000def' utf8_with_null = unicode_with_null.encode('utf-8') for false_argument in [None, '', invalid_utf8_str, unicode_with_null, utf8_with_null]: self.assertFalse(constraints.check_utf8(false_argument)) for true_argument in ['this is ascii and utf-8, too', unicode_sample, valid_utf8_str]: self.assertTrue(constraints.check_utf8(true_argument)) def test_validate_bad_meta(self): req = Request.blank( '/v/a/c/o', headers={'x-object-meta-hello': 'ab' * constraints.MAX_HEADER_SIZE}) self.assertEquals(constraints.check_metadata(req, 'object').status_int, HTTP_BAD_REQUEST) self.assertIn('x-object-meta-hello', constraints.check_metadata(req, 'object').body.lower()) def test_validate_constraints(self): c = constraints self.assertTrue(c.MAX_META_OVERALL_SIZE > c.MAX_META_NAME_LENGTH) self.assertTrue(c.MAX_META_OVERALL_SIZE > c.MAX_META_VALUE_LENGTH) self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_NAME_LENGTH) self.assertTrue(c.MAX_HEADER_SIZE > c.MAX_META_VALUE_LENGTH) def test_validate_copy_from(self): req = Request.blank( '/v/a/c/o', headers={'x-copy-from': 'c/o2'}) src_cont, src_obj = constraints.check_copy_from_header(req) self.assertEqual(src_cont, 'c') self.assertEqual(src_obj, 'o2') req = Request.blank( '/v/a/c/o', headers={'x-copy-from': 'c/subdir/o2'}) src_cont, src_obj = constraints.check_copy_from_header(req) self.assertEqual(src_cont, 'c') self.assertEqual(src_obj, 'subdir/o2') req = Request.blank( '/v/a/c/o', headers={'x-copy-from': '/c/o2'}) src_cont, src_obj = constraints.check_copy_from_header(req) self.assertEqual(src_cont, 'c') self.assertEqual(src_obj, 'o2') def test_validate_bad_copy_from(self): req = Request.blank( '/v/a/c/o', headers={'x-copy-from': 'bad_object'}) self.assertRaises(HTTPException, constraints.check_copy_from_header, req) class TestConstraintsConfig(unittest.TestCase): def test_default_constraints(self): for key in constraints.DEFAULT_CONSTRAINTS: # if there is local over-rides in swift.conf we just continue on if key in constraints.OVERRIDE_CONSTRAINTS: continue # module level attrs (that aren't in OVERRIDE) should have the # same value as the DEFAULT map module_level_value = getattr(constraints, key.upper()) self.assertEquals(constraints.DEFAULT_CONSTRAINTS[key], module_level_value) def test_effective_constraints(self): for key in constraints.DEFAULT_CONSTRAINTS: # module level attrs should always mirror the same value as the # EFFECTIVE map module_level_value = getattr(constraints, key.upper()) self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], module_level_value) # if there are local over-rides in swift.conf those should be # reflected in the EFFECTIVE, otherwise we expect the DEFAULTs self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], constraints.OVERRIDE_CONSTRAINTS.get( key, constraints.DEFAULT_CONSTRAINTS[key])) def test_override_constraints(self): try: with tempfile.NamedTemporaryFile() as f: f.write('[swift-constraints]\n') # set everything to 1 for key in constraints.DEFAULT_CONSTRAINTS: f.write('%s = 1\n' % key) f.flush() with mock.patch.object(constraints, 'SWIFT_CONF_FILE', f.name): constraints.reload_constraints() for key in constraints.DEFAULT_CONSTRAINTS: # module level attrs should all be 1 module_level_value = getattr(constraints, key.upper()) self.assertEquals(module_level_value, 1) # all keys should be in OVERRIDE self.assertEquals(constraints.OVERRIDE_CONSTRAINTS[key], module_level_value) # module level attrs should always mirror the same value as # the EFFECTIVE map self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS[key], module_level_value) finally: constraints.reload_constraints() def test_reload_reset(self): try: with tempfile.NamedTemporaryFile() as f: f.write('[swift-constraints]\n') # set everything to 1 for key in constraints.DEFAULT_CONSTRAINTS: f.write('%s = 1\n' % key) f.flush() with mock.patch.object(constraints, 'SWIFT_CONF_FILE', f.name): constraints.reload_constraints() self.assertTrue(constraints.SWIFT_CONSTRAINTS_LOADED) self.assertEquals(sorted(constraints.DEFAULT_CONSTRAINTS.keys()), sorted(constraints.OVERRIDE_CONSTRAINTS.keys())) # file is now deleted... with mock.patch.object(constraints, 'SWIFT_CONF_FILE', f.name): constraints.reload_constraints() # no constraints have been loaded from non-existant swift.conf self.assertFalse(constraints.SWIFT_CONSTRAINTS_LOADED) # no constraints are in OVERRIDE self.assertEquals([], constraints.OVERRIDE_CONSTRAINTS.keys()) # the EFFECTIVE constraints mirror DEFAULT self.assertEquals(constraints.EFFECTIVE_CONSTRAINTS, constraints.DEFAULT_CONSTRAINTS) finally: constraints.reload_constraints() if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_request_helpers.py0000664000175400017540000000536712323703611024151 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.common.request_helpers""" import unittest from swift.common.request_helpers import is_sys_meta, is_user_meta, \ is_sys_or_user_meta, strip_sys_meta_prefix, strip_user_meta_prefix, \ remove_items server_types = ['account', 'container', 'object'] class TestRequestHelpers(unittest.TestCase): def test_is_user_meta(self): m_type = 'meta' for st in server_types: self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type))) self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type))) self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type))) def test_is_sys_meta(self): m_type = 'sysmeta' for st in server_types: self.assertTrue(is_sys_meta(st, 'x-%s-%s-foo' % (st, m_type))) self.assertFalse(is_sys_meta(st, 'x-%s-%s-' % (st, m_type))) self.assertFalse(is_sys_meta(st, 'x-%s-%sfoo' % (st, m_type))) def test_is_sys_or_user_meta(self): m_types = ['sysmeta', 'meta'] for mt in m_types: for st in server_types: self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo' % (st, mt))) self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-' % (st, mt))) self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo' % (st, mt))) def test_strip_sys_meta_prefix(self): mt = 'sysmeta' for st in server_types: self.assertEquals(strip_sys_meta_prefix(st, 'x-%s-%s-a' % (st, mt)), 'a') def test_strip_user_meta_prefix(self): mt = 'meta' for st in server_types: self.assertEquals(strip_user_meta_prefix(st, 'x-%s-%s-a' % (st, mt)), 'a') def test_remove_items(self): src = {'a': 'b', 'c': 'd'} test = lambda x: x == 'a' rem = remove_items(src, test) self.assertEquals(src, {'c': 'd'}) self.assertEquals(rem, {'a': 'b'}) swift-1.13.1/test/unit/common/middleware/0000775000175400017540000000000012323703665021441 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/middleware/test_gatekeeper.py0000664000175400017540000001506112323703611025160 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request, Response from swift.common.middleware import gatekeeper class FakeApp(object): def __init__(self, headers={}): self.headers = headers self.req = None def __call__(self, env, start_response): self.req = Request(env) return Response(request=self.req, body='FAKE APP', headers=self.headers)(env, start_response) class FakeMiddleware(object): def __init__(self, app, conf, header_list=None): self.app = app self.conf = conf self.header_list = header_list def __call__(self, env, start_response): def fake_resp(status, response_headers, exc_info=None): for i in self.header_list: response_headers.append(i) return start_response(status, response_headers, exc_info) return self.app(env, fake_resp) class TestGatekeeper(unittest.TestCase): methods = ['PUT', 'POST', 'GET', 'DELETE', 'HEAD', 'COPY', 'OPTIONS'] allowed_headers = {'xx-account-sysmeta-foo': 'value', 'xx-container-sysmeta-foo': 'value', 'xx-object-sysmeta-foo': 'value', 'x-account-meta-foo': 'value', 'x-container-meta-foo': 'value', 'x-object-meta-foo': 'value', 'x-timestamp-foo': 'value'} sysmeta_headers = {'x-account-sysmeta-': 'value', 'x-container-sysmeta-': 'value', 'x-object-sysmeta-': 'value', 'x-account-sysmeta-foo': 'value', 'x-container-sysmeta-foo': 'value', 'x-object-sysmeta-foo': 'value', 'X-Account-Sysmeta-BAR': 'value', 'X-Container-Sysmeta-BAR': 'value', 'X-Object-Sysmeta-BAR': 'value'} x_backend_headers = {'X-Backend-Replication': 'true', 'X-Backend-Replication-Headers': 'stuff'} forbidden_headers_out = dict(sysmeta_headers.items() + x_backend_headers.items()) forbidden_headers_in = dict(sysmeta_headers.items() + x_backend_headers.items()) def _assertHeadersEqual(self, expected, actual): for key in expected: self.assertTrue(key.lower() in actual, '%s missing from %s' % (key, actual)) def _assertHeadersAbsent(self, unexpected, actual): for key in unexpected: self.assertTrue(key.lower() not in actual, '%s is in %s' % (key, actual)) def get_app(self, app, global_conf, **local_conf): factory = gatekeeper.filter_factory(global_conf, **local_conf) return factory(app) def test_ok_header(self): req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers=self.allowed_headers) fake_app = FakeApp() app = self.get_app(fake_app, {}) resp = req.get_response(app) self.assertEquals('200 OK', resp.status) self.assertEquals(resp.body, 'FAKE APP') self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) def _test_reserved_header_removed_inbound(self, method): headers = dict(self.forbidden_headers_in) headers.update(self.allowed_headers) req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}, headers=headers) fake_app = FakeApp() app = self.get_app(fake_app, {}) resp = req.get_response(app) self.assertEquals('200 OK', resp.status) self._assertHeadersEqual(self.allowed_headers, fake_app.req.headers) self._assertHeadersAbsent(self.forbidden_headers_in, fake_app.req.headers) def test_reserved_header_removed_inbound(self): for method in self.methods: self._test_reserved_header_removed_inbound(method) def _test_reserved_header_removed_outbound(self, method): headers = dict(self.forbidden_headers_out) headers.update(self.allowed_headers) req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}) fake_app = FakeApp(headers=headers) app = self.get_app(fake_app, {}) resp = req.get_response(app) self.assertEquals('200 OK', resp.status) self._assertHeadersEqual(self.allowed_headers, resp.headers) self._assertHeadersAbsent(self.forbidden_headers_out, resp.headers) def test_reserved_header_removed_outbound(self): for method in self.methods: self._test_reserved_header_removed_outbound(method) def _test_duplicate_headers_not_removed(self, method, app_hdrs): def fake_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) headers = [('X-Header', 'xxx'), ('X-Header', 'yyy')] def fake_filter(app): return FakeMiddleware(app, conf, headers) return fake_filter def fake_start_response(status, response_headers, exc_info=None): hdr_list = [] for k, v in response_headers: if k == 'X-Header': hdr_list.append(v) self.assertTrue('xxx' in hdr_list) self.assertTrue('yyy' in hdr_list) self.assertEqual(len(hdr_list), 2) req = Request.blank('/v/a/c', environ={'REQUEST_METHOD': method}) fake_app = FakeApp(headers=app_hdrs) factory = gatekeeper.filter_factory({}) factory_wrap = fake_factory({}) app = factory(factory_wrap(fake_app)) app(req.environ, fake_start_response) def test_duplicate_headers_not_removed(self): for method in self.methods: for app_hdrs in ({}, self.forbidden_headers_out): self._test_duplicate_headers_not_removed(method, app_hdrs) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_except.py0000664000175400017540000001000712323703611024327 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request from swift.common.middleware import catch_errors from swift.common.utils import get_logger class StrangeException(BaseException): pass class FakeApp(object): def __init__(self, error=False, body_iter=None): self.error = error self.body_iter = body_iter def __call__(self, env, start_response): if 'swift.trans_id' not in env: raise Exception('Trans id should always be in env') if self.error: if self.error == 'strange': raise StrangeException('whoa') raise Exception('An error occurred') if self.body_iter is None: return ["FAKE APP"] else: return self.body_iter def start_response(*args): pass class TestCatchErrors(unittest.TestCase): def setUp(self): self.logger = get_logger({}) self.logger.txn_id = None def test_catcherrors_passthrough(self): app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) self.assertEquals(list(resp), ['FAKE APP']) def test_catcherrors(self): app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) self.assertEquals(list(resp), ['An error occurred']) def test_trans_id_header_pass(self): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) req = Request.blank('/v1/a/c/o') app(req.environ, start_response) self.assertEquals(len(self.logger.txn_id), 34) # 32 hex + 'tx' def test_trans_id_header_fail(self): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) req = Request.blank('/v1/a/c/o') app(req.environ, start_response) self.assertEquals(len(self.logger.txn_id), 34) def test_error_in_iterator(self): app = catch_errors.CatchErrorMiddleware( FakeApp(body_iter=(int(x) for x in 'abcd')), {}) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) self.assertEquals(list(resp), ['An error occurred']) def test_trans_id_header_suffix(self): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware( FakeApp(), {'trans_id_suffix': '-stuff'}) req = Request.blank('/v1/a/c/o') app(req.environ, start_response) self.assertTrue(self.logger.txn_id.endswith('-stuff')) def test_catcherrors_with_unexpected_error(self): app = catch_errors.CatchErrorMiddleware(FakeApp(error='strange'), {}) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) self.assertEquals(list(resp), ['An error occurred']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_tempauth.py0000664000175400017540000014133712323703611024701 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from contextlib import contextmanager from base64 import b64encode from time import time from swift.common.middleware import tempauth as auth from swift.common.middleware.acl import format_acl from swift.common.swob import Request, Response from swift.common.utils import split_path NO_CONTENT_RESP = (('204 No Content', {}, ''),) # mock server response class FakeMemcache(object): def __init__(self): self.store = {} def get(self, key): return self.store.get(key) def set(self, key, value, time=0): self.store[key] = value return True def incr(self, key, time=0): self.store[key] = self.store.setdefault(key, 0) + 1 return self.store[key] @contextmanager def soft_lock(self, key, timeout=0, retries=5): yield True def delete(self, key): try: del self.store[key] except Exception: pass return True class FakeApp(object): def __init__(self, status_headers_body_iter=None, acl=None, sync_key=None): self.calls = 0 self.status_headers_body_iter = status_headers_body_iter if not self.status_headers_body_iter: self.status_headers_body_iter = iter([('404 Not Found', {}, '')]) self.acl = acl self.sync_key = sync_key def __call__(self, env, start_response): self.calls += 1 self.request = Request(env) if self.acl: self.request.acl = self.acl if self.sync_key: self.request.environ['swift_sync_key'] = self.sync_key if 'swift.authorize' in env: resp = env['swift.authorize'](self.request) if resp: return resp(env, start_response) status, headers, body = self.status_headers_body_iter.next() return Response(status=status, headers=headers, body=body)(env, start_response) class FakeConn(object): def __init__(self, status_headers_body_iter=None): self.calls = 0 self.status_headers_body_iter = status_headers_body_iter if not self.status_headers_body_iter: self.status_headers_body_iter = iter([('404 Not Found', {}, '')]) def request(self, method, path, headers): self.calls += 1 self.request_path = path self.status, self.headers, self.body = \ self.status_headers_body_iter.next() self.status, self.reason = self.status.split(' ', 1) self.status = int(self.status) def getresponse(self): return self def read(self): body = self.body self.body = '' return body class TestAuth(unittest.TestCase): def setUp(self): self.test_auth = auth.filter_factory({})(FakeApp()) def _make_request(self, path, **kwargs): req = Request.blank(path, **kwargs) req.environ['swift.cache'] = FakeMemcache() return req def test_reseller_prefix_init(self): app = FakeApp() ath = auth.filter_factory({})(app) self.assertEquals(ath.reseller_prefix, 'AUTH_') ath = auth.filter_factory({'reseller_prefix': 'TEST'})(app) self.assertEquals(ath.reseller_prefix, 'TEST_') ath = auth.filter_factory({'reseller_prefix': 'TEST_'})(app) self.assertEquals(ath.reseller_prefix, 'TEST_') def test_auth_prefix_init(self): app = FakeApp() ath = auth.filter_factory({})(app) self.assertEquals(ath.auth_prefix, '/auth/') ath = auth.filter_factory({'auth_prefix': ''})(app) self.assertEquals(ath.auth_prefix, '/auth/') ath = auth.filter_factory({'auth_prefix': '/'})(app) self.assertEquals(ath.auth_prefix, '/auth/') ath = auth.filter_factory({'auth_prefix': '/test/'})(app) self.assertEquals(ath.auth_prefix, '/test/') ath = auth.filter_factory({'auth_prefix': '/test'})(app) self.assertEquals(ath.auth_prefix, '/test/') ath = auth.filter_factory({'auth_prefix': 'test/'})(app) self.assertEquals(ath.auth_prefix, '/test/') ath = auth.filter_factory({'auth_prefix': 'test'})(app) self.assertEquals(ath.auth_prefix, '/test/') def test_top_level_deny(self): req = self._make_request('/') resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(req.environ['swift.authorize'], self.test_auth.denied_response) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="unknown"') def test_anon(self): req = self._make_request('/v1/AUTH_account') resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(req.environ['swift.authorize'], self.test_auth.authorize) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_account"') def test_anon_badpath(self): req = self._make_request('/v1') resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="unknown"') def test_override_asked_for_but_not_allowed(self): self.test_auth = \ auth.filter_factory({'allow_overrides': 'false'})(FakeApp()) req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_account"') self.assertEquals(req.environ['swift.authorize'], self.test_auth.authorize) def test_override_asked_for_and_allowed(self): self.test_auth = \ auth.filter_factory({'allow_overrides': 'true'})(FakeApp()) req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 404) self.assertTrue('swift.authorize' not in req.environ) def test_override_default_allowed(self): req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 404) self.assertTrue('swift.authorize' not in req.environ) def test_auth_deny_non_reseller_prefix(self): req = self._make_request('/v1/BLAH_account', headers={'X-Auth-Token': 'BLAH_t'}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="BLAH_account"') self.assertEquals(req.environ['swift.authorize'], self.test_auth.denied_response) def test_auth_deny_non_reseller_prefix_no_override(self): fake_authorize = lambda x: Response(status='500 Fake') req = self._make_request('/v1/BLAH_account', headers={'X-Auth-Token': 'BLAH_t'}, environ={'swift.authorize': fake_authorize} ) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 500) self.assertEquals(req.environ['swift.authorize'], fake_authorize) def test_auth_no_reseller_prefix_deny(self): # Ensures that when we have no reseller prefix, we don't deny a request # outright but set up a denial swift.authorize and pass the request on # down the chain. local_app = FakeApp() local_auth = auth.filter_factory({'reseller_prefix': ''})(local_app) req = self._make_request('/v1/account', headers={'X-Auth-Token': 't'}) resp = req.get_response(local_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="account"') self.assertEquals(local_app.calls, 1) self.assertEquals(req.environ['swift.authorize'], local_auth.denied_response) def test_auth_reseller_prefix_with_s3_deny(self): # Ensures that when we have a reseller prefix and using a middleware # relying on Http-Authorization (for example swift3), we don't deny a # request outright but set up a denial swift.authorize and pass the # request on down the chain. local_app = FakeApp() local_auth = auth.filter_factory({'reseller_prefix': 'PRE'})(local_app) req = self._make_request('/v1/account', headers={'X-Auth-Token': 't', 'Authorization': 'AWS user:pw'}) resp = req.get_response(local_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(local_app.calls, 1) self.assertEquals(req.environ['swift.authorize'], local_auth.denied_response) def test_auth_no_reseller_prefix_no_token(self): # Check that normally we set up a call back to our authorize. local_auth = auth.filter_factory({'reseller_prefix': ''})(FakeApp()) req = self._make_request('/v1/account') resp = req.get_response(local_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="account"') self.assertEquals(req.environ['swift.authorize'], local_auth.authorize) # Now make sure we don't override an existing swift.authorize when we # have no reseller prefix. local_auth = \ auth.filter_factory({'reseller_prefix': ''})(FakeApp()) local_authorize = lambda req: Response('test') req = self._make_request('/v1/account', environ={'swift.authorize': local_authorize}) resp = req.get_response(local_auth) self.assertEquals(resp.status_int, 200) self.assertEquals(req.environ['swift.authorize'], local_authorize) def test_auth_fail(self): resp = self._make_request( '/v1/AUTH_cfa', headers={'X-Auth-Token': 'AUTH_t'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') def test_authorize_bad_path(self): req = self._make_request('/badpath') resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="unknown"') req = self._make_request('/badpath') req.remote_user = 'act:usr,act,AUTH_cfa' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_authorize_account_access(self): req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act,AUTH_cfa' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_authorize_acl_group_access(self): self.test_auth = auth.filter_factory({})( FakeApp(iter(NO_CONTENT_RESP * 3))) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act2' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr2' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_deny_cross_reseller(self): # Tests that cross-reseller is denied, even if ACLs/group names match req = self._make_request('/v1/OTHER_cfa') req.remote_user = 'act:usr,act,AUTH_cfa' req.acl = 'act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_authorize_acl_referer_after_user_groups(self): req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr' req.acl = '.r:*,act:usr' self.assertEquals(self.test_auth.authorize(req), None) def test_authorize_acl_referrer_access(self): self.test_auth = auth.filter_factory({})( FakeApp(iter(NO_CONTENT_RESP * 6))) req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr,act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr,act' req.acl = '.r:*,.rlistings' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr,act' req.acl = '.r:*' # No listings allowed resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr,act' req.acl = '.r:.example.com,.rlistings' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_cfa/c') req.remote_user = 'act:usr,act' req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com,.rlistings' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa/c') resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') req = self._make_request('/v1/AUTH_cfa/c') req.acl = '.r:*,.rlistings' self.assertEquals(self.test_auth.authorize(req), None) req = self._make_request('/v1/AUTH_cfa/c') req.acl = '.r:*' # No listings allowed resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') req = self._make_request('/v1/AUTH_cfa/c') req.acl = '.r:.example.com,.rlistings' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') req = self._make_request('/v1/AUTH_cfa/c') req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com,.rlistings' self.assertEquals(self.test_auth.authorize(req), None) def test_detect_reseller_request(self): req = self._make_request('/v1/AUTH_admin', headers={'X-Auth-Token': 'AUTH_t'}) cache_key = 'AUTH_/token/AUTH_t' cache_entry = (time() + 3600, '.reseller_admin') req.environ['swift.cache'].set(cache_key, cache_entry) req.get_response(self.test_auth) self.assertTrue(req.environ.get('reseller_request', False)) def test_account_put_permissions(self): self.test_auth = auth.filter_factory({})( FakeApp(iter(NO_CONTENT_RESP * 4))) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_other' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) # Even PUTs to your own account as account admin should fail req = self._make_request('/v1/AUTH_old', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_old' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.reseller_admin' resp = self.test_auth.authorize(req) self.assertEquals(resp, None) # .super_admin is not something the middleware should ever see or care # about req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.super_admin' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_account_delete_permissions(self): self.test_auth = auth.filter_factory({})( FakeApp(iter(NO_CONTENT_RESP * 4))) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'DELETE'}) req.remote_user = 'act:usr,act' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'DELETE'}) req.remote_user = 'act:usr,act,AUTH_other' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) # Even DELETEs to your own account as account admin should fail req = self._make_request('/v1/AUTH_old', environ={'REQUEST_METHOD': 'DELETE'}) req.remote_user = 'act:usr,act,AUTH_old' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'DELETE'}) req.remote_user = 'act:usr,act,.reseller_admin' resp = self.test_auth.authorize(req) self.assertEquals(resp, None) # .super_admin is not something the middleware should ever see or care # about req = self._make_request('/v1/AUTH_new', environ={'REQUEST_METHOD': 'DELETE'}) req.remote_user = 'act:usr,act,.super_admin' resp = self.test_auth.authorize(req) self.assertEquals(resp.status_int, 403) def test_get_token_success(self): # Example of how to simulate the auth transaction test_auth = auth.filter_factory({'user_ac_user': 'testing'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 200) self.assertTrue(resp.headers['x-storage-url'].endswith('/v1/AUTH_ac')) self.assertTrue(resp.headers['x-auth-token'].startswith('AUTH_')) self.assertTrue(len(resp.headers['x-auth-token']) > 10) def test_use_token_success(self): # Example of how to simulate an authorized request test_auth = auth.filter_factory({'user_acct_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 1))) req = self._make_request('/v1/AUTH_acct', headers={'X-Auth-Token': 'AUTH_t'}) cache_key = 'AUTH_/token/AUTH_t' cache_entry = (time() + 3600, 'AUTH_acct') req.environ['swift.cache'].set(cache_key, cache_entry) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) def test_get_token_fail(self): resp = self._make_request('/auth/v1.0').get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="unknown"') resp = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'act:usr', 'X-Auth-Key': 'key'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertTrue('Www-Authenticate' in resp.headers) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="act"') def test_get_token_fail_invalid_x_auth_user_format(self): resp = self._make_request( '/auth/v1/act/auth', headers={'X-Auth-User': 'usr', 'X-Auth-Key': 'key'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="act"') def test_get_token_fail_non_matching_account_in_request(self): resp = self._make_request( '/auth/v1/act/auth', headers={'X-Auth-User': 'act2:usr', 'X-Auth-Key': 'key'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="act"') def test_get_token_fail_bad_path(self): resp = self._make_request( '/auth/v1/act/auth/invalid', headers={'X-Auth-User': 'act:usr', 'X-Auth-Key': 'key'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 400) def test_get_token_fail_missing_key(self): resp = self._make_request( '/auth/v1/act/auth', headers={'X-Auth-User': 'act:usr'}).get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="act"') def test_object_name_containing_slash(self): test_auth = auth.filter_factory({'user_acct_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 1))) req = self._make_request('/v1/AUTH_acct/cont/obj/name/with/slash', headers={'X-Auth-Token': 'AUTH_t'}) cache_key = 'AUTH_/token/AUTH_t' cache_entry = (time() + 3600, 'AUTH_acct') req.environ['swift.cache'].set(cache_key, cache_entry) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) def test_storage_url_default(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) del req.environ['HTTP_HOST'] req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['x-storage-url'], 'http://bob:1234/v1/AUTH_test') def test_storage_url_based_on_host(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) req.environ['HTTP_HOST'] = 'somehost:5678' req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['x-storage-url'], 'http://somehost:5678/v1/AUTH_test') def test_storage_url_overridden_scheme(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing', 'storage_url_scheme': 'fake'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) req.environ['HTTP_HOST'] = 'somehost:5678' req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['x-storage-url'], 'fake://somehost:5678/v1/AUTH_test') def test_use_old_token_from_memcached(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing', 'storage_url_scheme': 'fake'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) req.environ['HTTP_HOST'] = 'somehost:5678' req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' req.environ['swift.cache'].set('AUTH_/user/test:tester', 'uuid_token') req.environ['swift.cache'].set('AUTH_/token/uuid_token', (time() + 180, 'test,test:tester')) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['x-auth-token'], 'uuid_token') def test_old_token_overdate(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing', 'storage_url_scheme': 'fake'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) req.environ['HTTP_HOST'] = 'somehost:5678' req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' req.environ['swift.cache'].set('AUTH_/user/test:tester', 'uuid_token') req.environ['swift.cache'].set('AUTH_/token/uuid_token', (0, 'test,test:tester')) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertNotEquals(resp.headers['x-auth-token'], 'uuid_token') self.assertEquals(resp.headers['x-auth-token'][:7], 'AUTH_tk') def test_old_token_with_old_data(self): self.test_auth = \ auth.filter_factory({'user_test_tester': 'testing', 'storage_url_scheme': 'fake'})(FakeApp()) req = self._make_request( '/auth/v1.0', headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'}) req.environ['HTTP_HOST'] = 'somehost:5678' req.environ['SERVER_NAME'] = 'bob' req.environ['SERVER_PORT'] = '1234' req.environ['swift.cache'].set('AUTH_/user/test:tester', 'uuid_token') req.environ['swift.cache'].set('AUTH_/token/uuid_token', (time() + 99, 'test,test:tester,.role')) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 200) self.assertNotEquals(resp.headers['x-auth-token'], 'uuid_token') self.assertEquals(resp.headers['x-auth-token'][:7], 'AUTH_tk') def test_reseller_admin_is_owner(self): orig_authorize = self.test_auth.authorize owner_values = [] def mitm_authorize(req): rv = orig_authorize(req) owner_values.append(req.environ.get('swift_owner', False)) return rv self.test_auth.authorize = mitm_authorize req = self._make_request('/v1/AUTH_cfa', headers={'X-Auth-Token': 'AUTH_t'}) req.remote_user = '.reseller_admin' self.test_auth.authorize(req) self.assertEquals(owner_values, [True]) def test_admin_is_owner(self): orig_authorize = self.test_auth.authorize owner_values = [] def mitm_authorize(req): rv = orig_authorize(req) owner_values.append(req.environ.get('swift_owner', False)) return rv self.test_auth.authorize = mitm_authorize req = self._make_request( '/v1/AUTH_cfa', headers={'X-Auth-Token': 'AUTH_t'}) req.remote_user = 'AUTH_cfa' self.test_auth.authorize(req) self.assertEquals(owner_values, [True]) def test_regular_is_not_owner(self): orig_authorize = self.test_auth.authorize owner_values = [] def mitm_authorize(req): rv = orig_authorize(req) owner_values.append(req.environ.get('swift_owner', False)) return rv self.test_auth.authorize = mitm_authorize req = self._make_request( '/v1/AUTH_cfa/c', headers={'X-Auth-Token': 'AUTH_t'}) req.remote_user = 'act:usr' self.test_auth.authorize(req) self.assertEquals(owner_values, [False]) def test_sync_request_success(self): self.test_auth.app = FakeApp(iter(NO_CONTENT_RESP * 1), sync_key='secret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret', 'x-timestamp': '123.456'}) req.remote_addr = '127.0.0.1' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 204) def test_sync_request_fail_key(self): self.test_auth.app = FakeApp(sync_key='secret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'wrongsecret', 'x-timestamp': '123.456'}) req.remote_addr = '127.0.0.1' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') self.test_auth.app = FakeApp(sync_key='othersecret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret', 'x-timestamp': '123.456'}) req.remote_addr = '127.0.0.1' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') self.test_auth.app = FakeApp(sync_key=None) req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret', 'x-timestamp': '123.456'}) req.remote_addr = '127.0.0.1' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') def test_sync_request_fail_no_timestamp(self): self.test_auth.app = FakeApp(sync_key='secret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret'}) req.remote_addr = '127.0.0.1' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="AUTH_cfa"') def test_sync_request_success_lb_sync_host(self): self.test_auth.app = FakeApp(iter(NO_CONTENT_RESP * 1), sync_key='secret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret', 'x-timestamp': '123.456', 'x-forwarded-for': '127.0.0.1'}) req.remote_addr = '127.0.0.2' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 204) self.test_auth.app = FakeApp(iter(NO_CONTENT_RESP * 1), sync_key='secret') req = self._make_request( '/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'x-container-sync-key': 'secret', 'x-timestamp': '123.456', 'x-cluster-client-ip': '127.0.0.1'}) req.remote_addr = '127.0.0.2' resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 204) def test_options_call(self): req = self._make_request('/v1/AUTH_cfa/c/o', environ={'REQUEST_METHOD': 'OPTIONS'}) resp = self.test_auth.authorize(req) self.assertEquals(resp, None) def test_get_user_group(self): app = FakeApp() ath = auth.filter_factory({})(app) ath.users = {'test:tester': {'groups': ['.admin']}} groups = ath._get_user_groups('test', 'test:tester', 'AUTH_test') self.assertEquals(groups, 'test,test:tester,AUTH_test') ath.users = {'test:tester': {'groups': []}} groups = ath._get_user_groups('test', 'test:tester', 'AUTH_test') self.assertEquals(groups, 'test,test:tester') def test_auth_scheme(self): req = self._make_request('/v1/BLAH_account', headers={'X-Auth-Token': 'BLAH_t'}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) self.assertTrue('Www-Authenticate' in resp.headers) self.assertEquals(resp.headers.get('Www-Authenticate'), 'Swift realm="BLAH_account"') class TestParseUserCreation(unittest.TestCase): def test_parse_user_creation(self): auth_filter = auth.filter_factory({ 'reseller_prefix': 'ABC', 'user_test_tester3': 'testing', 'user_has_url': 'urlly .admin http://a.b/v1/DEF_has', 'user_admin_admin': 'admin .admin .reseller_admin', })(FakeApp()) self.assertEquals(auth_filter.users, { 'admin:admin': { 'url': '$HOST/v1/ABC_admin', 'groups': ['.admin', '.reseller_admin'], 'key': 'admin' }, 'test:tester3': { 'url': '$HOST/v1/ABC_test', 'groups': [], 'key': 'testing' }, 'has:url': { 'url': 'http://a.b/v1/DEF_has', 'groups': ['.admin'], 'key': 'urlly' }, }) def test_base64_encoding(self): auth_filter = auth.filter_factory({ 'reseller_prefix': 'ABC', 'user64_%s_%s' % ( b64encode('test').rstrip('='), b64encode('tester3').rstrip('=')): 'testing .reseller_admin', 'user64_%s_%s' % ( b64encode('user_foo').rstrip('='), b64encode('ab').rstrip('=')): 'urlly .admin http://a.b/v1/DEF_has', })(FakeApp()) self.assertEquals(auth_filter.users, { 'test:tester3': { 'url': '$HOST/v1/ABC_test', 'groups': ['.reseller_admin'], 'key': 'testing' }, 'user_foo:ab': { 'url': 'http://a.b/v1/DEF_has', 'groups': ['.admin'], 'key': 'urlly' }, }) def test_key_with_no_value(self): self.assertRaises(ValueError, auth.filter_factory({ 'user_test_tester3': 'testing', 'user_bob_bobby': '', 'user_admin_admin': 'admin .admin .reseller_admin', }), FakeApp()) class TestAccountAcls(unittest.TestCase): def _make_request(self, path, **kwargs): # Our TestAccountAcls default request will have a valid auth token version, acct, _ = split_path(path, 1, 3, True) headers = kwargs.pop('headers', {'X-Auth-Token': 'AUTH_t'}) user_groups = kwargs.pop('user_groups', 'AUTH_firstacct') # The account being accessed will have account ACLs acl = {'admin': ['AUTH_admin'], 'read-write': ['AUTH_rw'], 'read-only': ['AUTH_ro']} header_data = {'core-access-control': format_acl(version=2, acl_dict=acl)} acls = kwargs.pop('acls', header_data) req = Request.blank(path, headers=headers, **kwargs) # Authorize the token by populating the request's cache req.environ['swift.cache'] = FakeMemcache() cache_key = 'AUTH_/token/AUTH_t' cache_entry = (time() + 3600, user_groups) req.environ['swift.cache'].set(cache_key, cache_entry) # Pretend get_account_info returned ACLs in sysmeta, and we cached that cache_key = 'account/%s' % acct cache_entry = {'sysmeta': acls} req.environ['swift.cache'].set(cache_key, cache_entry) return req def test_account_acl_success(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 1))) # admin (not a swift admin) wants to read from otheracct req = self._make_request('/v1/AUTH_otheract', user_groups="AUTH_admin") # The request returned by _make_request should be allowed resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) def test_account_acl_failures(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp()) # If I'm not authed as anyone on the ACLs, I shouldn't get in req = self._make_request('/v1/AUTH_otheract', user_groups="AUTH_bob") resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 403) # If the target account has no ACLs, a non-owner shouldn't get in req = self._make_request('/v1/AUTH_otheract', user_groups="AUTH_admin", acls={}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 403) def test_admin_privileges(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 18))) for target in ('/v1/AUTH_otheracct', '/v1/AUTH_otheracct/container', '/v1/AUTH_otheracct/container/obj'): for method in ('GET', 'HEAD', 'OPTIONS', 'PUT', 'POST', 'DELETE'): # Admin ACL user can do anything req = self._make_request(target, user_groups="AUTH_admin", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) # swift_owner should be set to True if method != 'OPTIONS': self.assertTrue(req.environ.get('swift_owner')) def test_readwrite_privileges(self): test_auth = auth.filter_factory({'user_rw_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 15))) for target in ('/v1/AUTH_otheracct',): for method in ('GET', 'HEAD', 'OPTIONS'): # Read-Write user can read account data req = self._make_request(target, user_groups="AUTH_rw", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) # swift_owner should NOT be set to True self.assertFalse(req.environ.get('swift_owner')) # RW user should NOT be able to PUT, POST, or DELETE to the account for method in ('PUT', 'POST', 'DELETE'): req = self._make_request(target, user_groups="AUTH_rw", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 403) # RW user should be able to GET, PUT, POST, or DELETE to containers # and objects for target in ('/v1/AUTH_otheracct/c', '/v1/AUTH_otheracct/c/o'): for method in ('GET', 'HEAD', 'OPTIONS', 'PUT', 'POST', 'DELETE'): req = self._make_request(target, user_groups="AUTH_rw", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) def test_readonly_privileges(self): test_auth = auth.filter_factory({'user_ro_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 9))) # ReadOnly user should NOT be able to PUT, POST, or DELETE to account, # container, or object for target in ('/v1/AUTH_otheracct', '/v1/AUTH_otheracct/cont', '/v1/AUTH_otheracct/cont/obj'): for method in ('GET', 'HEAD', 'OPTIONS'): req = self._make_request(target, user_groups="AUTH_ro", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) # swift_owner should NOT be set to True for the ReadOnly ACL self.assertFalse(req.environ.get('swift_owner')) for method in ('PUT', 'POST', 'DELETE'): req = self._make_request(target, user_groups="AUTH_ro", environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 403) # swift_owner should NOT be set to True for the ReadOnly ACL self.assertFalse(req.environ.get('swift_owner')) def test_user_gets_best_acl(self): test_auth = auth.filter_factory({'user_acct_username': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 18))) mygroups = "AUTH_acct,AUTH_ro,AUTH_something,AUTH_admin" for target in ('/v1/AUTH_otheracct', '/v1/AUTH_otheracct/container', '/v1/AUTH_otheracct/container/obj'): for method in ('GET', 'HEAD', 'OPTIONS', 'PUT', 'POST', 'DELETE'): # Admin ACL user can do anything req = self._make_request(target, user_groups=mygroups, environ={'REQUEST_METHOD': method}) resp = req.get_response(test_auth) self.assertEquals( resp.status_int, 204, "%s (%s) - expected 204, got %d" % (target, method, resp.status_int)) # swift_owner should be set to True if method != 'OPTIONS': self.assertTrue(req.environ.get('swift_owner')) def test_acl_syntax_verification(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 5))) good_headers = {'X-Auth-Token': 'AUTH_t'} good_acl = '{"read-only":["a","b"]}' bad_acl = 'syntactically invalid acl -- this does not parse as JSON' wrong_acl = '{"other-auth-system":["valid","json","but","wrong"]}' bad_value_acl = '{"read-write":["fine"],"admin":"should be a list"}' not_dict_acl = '["read-only"]' not_dict_acl2 = 1 empty_acls = ['{}', '', '{ }'] target = '/v1/AUTH_firstacct' # no acls -- no problem! req = self._make_request(target, headers=good_headers) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) # syntactically valid acls should go through update = {'x-account-access-control': good_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) # syntactically valid empty acls should go through for acl in empty_acls: update = {'x-account-access-control': acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) errmsg = 'X-Account-Access-Control invalid: %s' # syntactically invalid acls get a 400 update = {'x-account-access-control': bad_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) self.assertEquals(errmsg % "Syntax error", resp.body[:46]) # syntactically valid acls with bad keys also get a 400 update = {'x-account-access-control': wrong_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) self.assertEquals(errmsg % "Key '", resp.body[:39]) # acls with good keys but bad values also get a 400 update = {'x-account-access-control': bad_value_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) self.assertEquals(errmsg % "Value", resp.body[:39]) # acls with wrong json structure also get a 400 update = {'x-account-access-control': not_dict_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) self.assertEquals(errmsg % "Syntax error", resp.body[:46]) # acls with wrong json structure also get a 400 update = {'x-account-access-control': not_dict_acl2} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) self.assertEquals(errmsg % "Syntax error", resp.body[:46]) def test_acls_propagate_to_sysmeta(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 3))) sysmeta_hdr = 'x-account-sysmeta-core-access-control' target = '/v1/AUTH_firstacct' good_headers = {'X-Auth-Token': 'AUTH_t'} good_acl = '{"read-only":["a","b"]}' # no acls -- no problem! req = self._make_request(target, headers=good_headers) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) self.assertEqual(None, req.headers.get(sysmeta_hdr)) # syntactically valid acls should go through update = {'x-account-access-control': good_acl} req = self._make_request(target, headers=dict(good_headers, **update)) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 204) self.assertEqual(good_acl, req.headers.get(sysmeta_hdr)) def test_bad_acls_get_denied(self): test_auth = auth.filter_factory({'user_admin_user': 'testing'})( FakeApp(iter(NO_CONTENT_RESP * 3))) target = '/v1/AUTH_firstacct' good_headers = {'X-Auth-Token': 'AUTH_t'} bad_acls = ( 'syntax error', '{"bad_key":"should_fail"}', '{"admin":"not a list, should fail"}', '{"admin":["valid"],"read-write":"not a list, should fail"}', ) for bad_acl in bad_acls: hdrs = dict(good_headers, **{'x-account-access-control': bad_acl}) req = self._make_request(target, headers=hdrs) resp = req.get_response(test_auth) self.assertEquals(resp.status_int, 400) class TestUtilityMethods(unittest.TestCase): def test_account_acls_bad_path_raises_exception(self): auth_inst = auth.filter_factory({})(FakeApp()) req = Request({'PATH_INFO': '/'}) self.assertRaises(ValueError, auth_inst.account_acls, req) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_quotas.py0000664000175400017540000003134412323703611024362 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request, HTTPUnauthorized from swift.common.middleware import container_quotas class FakeCache(object): def __init__(self, val): if 'status' not in val: val['status'] = 200 self.val = val def get(self, *args): return self.val class FakeApp(object): def __init__(self): pass def __call__(self, env, start_response): start_response('200 OK', []) return [] class FakeMissingApp(object): def __init__(self): pass def __call__(self, env, start_response): start_response('404 Not Found', []) return [] def start_response(*args): pass class TestContainerQuotas(unittest.TestCase): def test_split_path_empty_container_path_segment(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) req = Request.blank('/v1/a//something/something_else', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': {'key': 'value'}}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_handled(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_no_quotas(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': FakeCache({}), 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_bytes_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_exceed_bytes_quota_copy_from(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_exceed_bytes_quota_copy_verb(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '2'}}) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_not_exceed_bytes_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_bytes_quota_copy_from(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_bytes_quota_copy_verb(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_bytes_quota_copy_from_no_src(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'x-copy-from': '/c2/o3'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_bytes_quota_copy_from_bad_src(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': 'bad_path'}) res = req.get_response(app) self.assertEquals(res.status_int, 412) def test_bytes_quota_copy_verb_no_src(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank('/v1/a/c2/o3', environ={'REQUEST_METHOD': 'COPY', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_exceed_counts_quota_copy_from(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.object/a/c2/o2': {'length': 10}, 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_exceed_counts_quota_copy_verb(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}}) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_not_exceed_counts_quota(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_counts_quota_copy_from(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_counts_quota_copy_verb(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '2'}}) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_invalid_quotas(self): req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_QUOTA_BYTES': 'abc'}) res = req.get_response( container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) self.assertEquals(res.status_int, 400) req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_QUOTA_COUNT': 'abc'}) res = req.get_response( container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) self.assertEquals(res.status_int, 400) def test_valid_quotas(self): req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_QUOTA_BYTES': '123'}) res = req.get_response( container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) self.assertEquals(res.status_int, 200) req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_QUOTA_COUNT': '123'}) res = req.get_response( container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) self.assertEquals(res.status_int, 200) def test_delete_quotas(self): req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_QUOTA_BYTES': None}) res = req.get_response( container_quotas.ContainerQuotaMiddleware(FakeApp(), {})) self.assertEquals(res.status_int, 200) def test_missing_container(self): app = container_quotas.ContainerQuotaMiddleware(FakeMissingApp(), {}) cache = FakeCache({'bytes': 0, 'meta': {'quota-bytes': '100'}}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 404) def test_auth_fail(self): app = container_quotas.ContainerQuotaMiddleware(FakeApp(), {}) cache = FakeCache({'object_count': 1, 'meta': {'quota-count': '1'}, 'write_acl': None}) req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'CONTENT_LENGTH': '100', 'swift.authorize': lambda *args: HTTPUnauthorized()}) res = req.get_response(app) self.assertEquals(res.status_int, 401) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_proxy_logging.py0000664000175400017540000010674312323703611025743 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from urllib import unquote import cStringIO as StringIO from logging.handlers import SysLogHandler import mock from test.unit import FakeLogger from swift.common.utils import get_logger from swift.common.middleware import proxy_logging from swift.common.swob import Request, Response class FakeApp(object): def __init__(self, body=['FAKE APP'], response_str='200 OK'): self.body = body self.response_str = response_str def __call__(self, env, start_response): start_response(self.response_str, [('Content-Type', 'text/plain'), ('Content-Length', str(sum(map(len, self.body))))]) while env['wsgi.input'].read(5): pass return self.body class FakeAppThatExcepts(object): def __call__(self, env, start_response): raise Exception("We take exception to that!") class FakeAppNoContentLengthNoTransferEncoding(object): def __init__(self, body=['FAKE APP']): self.body = body def __call__(self, env, start_response): start_response('200 OK', [('Content-Type', 'text/plain')]) while env['wsgi.input'].read(5): pass return self.body class FileLikeExceptor(object): def __init__(self): pass def read(self, len): raise IOError('of some sort') def readline(self, len=1024): raise IOError('of some sort') class FakeAppReadline(object): def __call__(self, env, start_response): start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '8')]) env['wsgi.input'].readline() return ["FAKE APP"] def start_response(*args): pass class TestProxyLogging(unittest.TestCase): def _log_parts(self, app, should_be_empty=False): info_calls = app.access_logger.log_dict['info'] if should_be_empty: self.assertEquals([], info_calls) else: self.assertEquals(1, len(info_calls)) return info_calls[0][0][0].split(' ') def assertTiming(self, exp_metric, app, exp_timing=None): timing_calls = app.access_logger.log_dict['timing'] found = False for timing_call in timing_calls: self.assertEquals({}, timing_call[1]) self.assertEquals(2, len(timing_call[0])) if timing_call[0][0] == exp_metric: found = True if exp_timing is not None: self.assertAlmostEqual(exp_timing, timing_call[0][1], places=4) if not found: self.assertTrue(False, 'assertTiming: %s not found in %r' % ( exp_metric, timing_calls)) def assertTimingSince(self, exp_metric, app, exp_start=None): timing_calls = app.access_logger.log_dict['timing_since'] found = False for timing_call in timing_calls: self.assertEquals({}, timing_call[1]) self.assertEquals(2, len(timing_call[0])) if timing_call[0][0] == exp_metric: found = True if exp_start is not None: self.assertAlmostEqual(exp_start, timing_call[0][1], places=4) if not found: self.assertTrue(False, 'assertTimingSince: %s not found in %r' % ( exp_metric, timing_calls)) def assertNotTiming(self, not_exp_metric, app): timing_calls = app.access_logger.log_dict['timing'] for timing_call in timing_calls: self.assertNotEqual(not_exp_metric, timing_call[0][0]) def assertUpdateStats(self, exp_metric, exp_bytes, app): update_stats_calls = app.access_logger.log_dict['update_stats'] self.assertEquals(1, len(update_stats_calls)) self.assertEquals({}, update_stats_calls[0][1]) self.assertEquals((exp_metric, exp_bytes), update_stats_calls[0][0]) def test_log_request_statsd_invalid_stats_types(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() for url in ['/', '/foo', '/foo/bar', '/v1']: req = Request.blank(url, environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) # get body ''.join(resp) self.assertEqual([], app.access_logger.log_dict['timing']) self.assertEqual([], app.access_logger.log_dict['update_stats']) def test_log_request_stat_type_bad(self): for bad_path in ['', '/', '/bad', '/baddy/mc_badderson', '/v1', '/v1/']: app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank(bad_path, environ={'REQUEST_METHOD': 'GET'}) now = 10000.0 app.log_request(req, 123, 7, 13, now, now + 2.71828182846) self.assertEqual([], app.access_logger.log_dict['timing']) self.assertEqual([], app.access_logger.log_dict['update_stats']) def test_log_request_stat_type_good(self): """ log_request() should send timing and byte-count counters for GET requests. Also, __call__()'s iter_response() function should statsd-log time to first byte (calling the passed-in start_response function), but only for GET requests. """ stub_times = [] def stub_time(): return stub_times.pop(0) path_types = { '/v1/a': 'account', '/v1/a/': 'account', '/v1/a/c': 'container', '/v1/a/c/': 'container', '/v1/a/c/o': 'object', '/v1/a/c/o/': 'object', '/v1/a/c/o/p': 'object', '/v1/a/c/o/p/': 'object', '/v1/a/c/o/p/p2': 'object', } with mock.patch("time.time", stub_time): for path, exp_type in path_types.iteritems(): # GET app = proxy_logging.ProxyLoggingMiddleware( FakeApp(body='7654321', response_str='321 Fubar'), {}) app.access_logger = FakeLogger() req = Request.blank(path, environ={ 'REQUEST_METHOD': 'GET', 'wsgi.input': StringIO.StringIO('4321')}) stub_times = [18.0, 20.71828182846] iter_response = app(req.environ, lambda *_: None) self.assertEqual('7654321', ''.join(iter_response)) self.assertTiming('%s.GET.321.timing' % exp_type, app, exp_timing=2.71828182846 * 1000) self.assertTimingSince( '%s.GET.321.first-byte.timing' % exp_type, app, exp_start=18.0) self.assertUpdateStats('%s.GET.321.xfer' % exp_type, 4 + 7, app) # GET with swift.proxy_access_log_made already set app = proxy_logging.ProxyLoggingMiddleware( FakeApp(body='7654321', response_str='321 Fubar'), {}) app.access_logger = FakeLogger() req = Request.blank(path, environ={ 'REQUEST_METHOD': 'GET', 'swift.proxy_access_log_made': True, 'wsgi.input': StringIO.StringIO('4321')}) stub_times = [18.0, 20.71828182846] iter_response = app(req.environ, lambda *_: None) self.assertEqual('7654321', ''.join(iter_response)) self.assertEqual([], app.access_logger.log_dict['timing']) self.assertEqual([], app.access_logger.log_dict['timing_since']) self.assertEqual([], app.access_logger.log_dict['update_stats']) # PUT (no first-byte timing!) app = proxy_logging.ProxyLoggingMiddleware( FakeApp(body='87654321', response_str='314 PiTown'), {}) app.access_logger = FakeLogger() req = Request.blank(path, environ={ 'REQUEST_METHOD': 'PUT', 'wsgi.input': StringIO.StringIO('654321')}) # (it's not a GET, so time() doesn't have a 2nd call) stub_times = [58.2, 58.2 + 7.3321] iter_response = app(req.environ, lambda *_: None) self.assertEqual('87654321', ''.join(iter_response)) self.assertTiming('%s.PUT.314.timing' % exp_type, app, exp_timing=7.3321 * 1000) self.assertNotTiming( '%s.GET.314.first-byte.timing' % exp_type, app) self.assertNotTiming( '%s.PUT.314.first-byte.timing' % exp_type, app) self.assertUpdateStats( '%s.PUT.314.xfer' % exp_type, 6 + 8, app) def test_log_request_stat_method_filtering_default(self): method_map = { 'foo': 'BAD_METHOD', '': 'BAD_METHOD', 'PUTT': 'BAD_METHOD', 'SPECIAL': 'BAD_METHOD', 'GET': 'GET', 'PUT': 'PUT', 'COPY': 'COPY', 'HEAD': 'HEAD', 'POST': 'POST', 'DELETE': 'DELETE', 'OPTIONS': 'OPTIONS', } for method, exp_method in method_map.iteritems(): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/v1/a/', environ={'REQUEST_METHOD': method}) now = 10000.0 app.log_request(req, 299, 11, 3, now, now + 1.17) self.assertTiming('account.%s.299.timing' % exp_method, app, exp_timing=1.17 * 1000) self.assertUpdateStats('account.%s.299.xfer' % exp_method, 11 + 3, app) def test_log_request_stat_method_filtering_custom(self): method_map = { 'foo': 'BAD_METHOD', '': 'BAD_METHOD', 'PUTT': 'BAD_METHOD', 'SPECIAL': 'SPECIAL', # will be configured 'GET': 'GET', 'PUT': 'PUT', 'COPY': 'BAD_METHOD', # prove no one's special } # this conf var supports optional leading access_ for conf_key in ['access_log_statsd_valid_http_methods', 'log_statsd_valid_http_methods']: for method, exp_method in method_map.iteritems(): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { conf_key: 'SPECIAL, GET,PUT ', # crazy spaces ok }) app.access_logger = FakeLogger() req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}) now = 10000.0 app.log_request(req, 911, 4, 43, now, now + 1.01) self.assertTiming('container.%s.911.timing' % exp_method, app, exp_timing=1.01 * 1000) self.assertUpdateStats('container.%s.911.xfer' % exp_method, 4 + 43, app) def test_basic_req(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'FAKE APP') self.assertEquals(log_parts[11], str(len(resp_body))) def test_basic_req_second_time(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={ 'swift.proxy_access_log_made': True, 'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) self._log_parts(app, should_be_empty=True) self.assertEquals(resp_body, 'FAKE APP') def test_multi_segment_resp(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp( ['some', 'chunks', 'of data']), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.source': 'SOS'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'somechunksof data') self.assertEquals(log_parts[11], str(len(resp_body))) self.assertUpdateStats('SOS.GET.200.xfer', len(resp_body), app) def test_log_headers(self): for conf_key in ['access_log_headers', 'log_headers']: app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {conf_key: 'yes'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) headers = unquote(log_parts[14]).split('\n') self.assert_('Host: localhost:80' in headers) def test_access_log_headers_only(self): app = proxy_logging.ProxyLoggingMiddleware( FakeApp(), {'log_headers': 'yes', 'access_log_headers_only': 'FIRST, seCond'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'First': '1', 'Second': '2', 'Third': '3'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) headers = unquote(log_parts[14]).split('\n') self.assert_('First: 1' in headers) self.assert_('Second: 2' in headers) self.assert_('Third: 3' not in headers) self.assert_('Host: localhost:80' not in headers) def test_upload_size(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {'log_headers': 'yes'}) app.access_logger = FakeLogger() req = Request.blank( '/v1/a/c/o/foo', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': StringIO.StringIO('some stuff')}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[11], str(len('FAKE APP'))) self.assertEquals(log_parts[10], str(len('some stuff'))) self.assertUpdateStats('object.PUT.200.xfer', len('some stuff') + len('FAKE APP'), app) def test_upload_line(self): app = proxy_logging.ProxyLoggingMiddleware(FakeAppReadline(), {'log_headers': 'yes'}) app.access_logger = FakeLogger() req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO.StringIO( 'some stuff\nsome other stuff\n')}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[11], str(len('FAKE APP'))) self.assertEquals(log_parts[10], str(len('some stuff\n'))) self.assertUpdateStats('container.POST.200.xfer', len('some stuff\n') + len('FAKE APP'), app) def test_log_query_string(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'x=3'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(unquote(log_parts[4]), '/?x=3') def test_client_logging(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'REMOTE_ADDR': '1.2.3.4'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], '1.2.3.4') # client ip self.assertEquals(log_parts[1], '1.2.3.4') # remote addr def test_proxy_client_logging(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={ 'REQUEST_METHOD': 'GET', 'REMOTE_ADDR': '1.2.3.4', 'HTTP_X_FORWARDED_FOR': '4.5.6.7,8.9.10.11'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], '4.5.6.7') # client ip self.assertEquals(log_parts[1], '1.2.3.4') # remote addr app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={ 'REQUEST_METHOD': 'GET', 'REMOTE_ADDR': '1.2.3.4', 'HTTP_X_CLUSTER_CLIENT_IP': '4.5.6.7'}) resp = app(req.environ, start_response) # exhaust generator [x for x in resp] log_parts = self._log_parts(app) self.assertEquals(log_parts[0], '4.5.6.7') # client ip self.assertEquals(log_parts[1], '1.2.3.4') # remote addr def test_facility(self): app = proxy_logging.ProxyLoggingMiddleware( FakeApp(), {'log_headers': 'yes', 'access_log_facility': 'LOG_LOCAL7'}) handler = get_logger.handler4logger[app.access_logger.logger] self.assertEquals(SysLogHandler.LOG_LOCAL7, handler.facility) def test_filter(self): factory = proxy_logging.filter_factory({}) self.assert_(callable(factory)) self.assert_(callable(factory(FakeApp()))) def test_unread_body(self): app = proxy_logging.ProxyLoggingMiddleware( FakeApp(['some', 'stuff']), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) # read first chunk next(resp) resp.close() # raise a GeneratorExit in middleware app_iter loop log_parts = self._log_parts(app) self.assertEquals(log_parts[6], '499') self.assertEquals(log_parts[11], '4') # write length def test_disconnect_on_readline(self): app = proxy_logging.ProxyLoggingMiddleware(FakeAppReadline(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'wsgi.input': FileLikeExceptor()}) try: resp = app(req.environ, start_response) # read body ''.join(resp) except IOError: pass log_parts = self._log_parts(app) self.assertEquals(log_parts[6], '499') self.assertEquals(log_parts[10], '-') # read length def test_disconnect_on_read(self): app = proxy_logging.ProxyLoggingMiddleware( FakeApp(['some', 'stuff']), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'wsgi.input': FileLikeExceptor()}) try: resp = app(req.environ, start_response) # read body ''.join(resp) except IOError: pass log_parts = self._log_parts(app) self.assertEquals(log_parts[6], '499') self.assertEquals(log_parts[10], '-') # read length def test_app_exception(self): app = proxy_logging.ProxyLoggingMiddleware( FakeAppThatExcepts(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) try: app(req.environ, start_response) except Exception: pass log_parts = self._log_parts(app) self.assertEquals(log_parts[6], '500') self.assertEquals(log_parts[10], '-') # read length def test_no_content_length_no_transfer_encoding_with_list_body(self): app = proxy_logging.ProxyLoggingMiddleware( FakeAppNoContentLengthNoTransferEncoding( # test the "while not chunk: chunk = iterator.next()" body=['', '', 'line1\n', 'line2\n'], ), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'line1\nline2\n') self.assertEquals(log_parts[11], str(len(resp_body))) def test_no_content_length_no_transfer_encoding_with_empty_strings(self): app = proxy_logging.ProxyLoggingMiddleware( FakeAppNoContentLengthNoTransferEncoding( # test the "while not chunk: chunk = iterator.next()" body=['', '', ''], ), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, '') self.assertEquals(log_parts[11], '-') def test_no_content_length_no_transfer_encoding_with_generator(self): class BodyGen(object): def __init__(self, data): self.data = data def __iter__(self): yield self.data app = proxy_logging.ProxyLoggingMiddleware( FakeAppNoContentLengthNoTransferEncoding( body=BodyGen('abc'), ), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'abc') self.assertEquals(log_parts[11], '3') def test_req_path_info_popping(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/v1/something', environ={'REQUEST_METHOD': 'GET'}) req.path_info_pop() self.assertEquals(req.environ['PATH_INFO'], '/something') resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/v1/something') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'FAKE APP') self.assertEquals(log_parts[11], str(len(resp_body))) def test_ipv6(self): ipv6addr = '2001:db8:85a3:8d3:1319:8a2e:370:7348' app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) req.remote_addr = ipv6addr resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[0], ipv6addr) self.assertEquals(log_parts[1], ipv6addr) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'FAKE APP') self.assertEquals(log_parts[11], str(len(resp_body))) def test_log_info_none(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) list(app(req.environ, start_response)) log_parts = self._log_parts(app) self.assertEquals(log_parts[17], '-') app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) req.environ['swift.log_info'] = [] list(app(req.environ, start_response)) log_parts = self._log_parts(app) self.assertEquals(log_parts[17], '-') def test_log_info_single(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) req.environ['swift.log_info'] = ['one'] list(app(req.environ, start_response)) log_parts = self._log_parts(app) self.assertEquals(log_parts[17], 'one') def test_log_info_multiple(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) req.environ['swift.log_info'] = ['one', 'and two'] list(app(req.environ, start_response)) log_parts = self._log_parts(app) self.assertEquals(log_parts[17], 'one%2Cand%20two') def test_log_auth_token(self): auth_token = 'b05bf940-0464-4c0e-8c70-87717d2d73e8' # Default - no reveal_sensitive_prefix in config # No x-auth-token header app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], '-') # Has x-auth-token header app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_AUTH_TOKEN': auth_token}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], auth_token) # Truncate to first 8 characters app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { 'reveal_sensitive_prefix': '8'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], '-') app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { 'reveal_sensitive_prefix': '8'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_AUTH_TOKEN': auth_token}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], 'b05bf940...') # Token length and reveal_sensitive_prefix are same (no truncate) app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { 'reveal_sensitive_prefix': str(len(auth_token))}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_AUTH_TOKEN': auth_token}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], auth_token) # Don't log x-auth-token app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { 'reveal_sensitive_prefix': '0'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], '-') app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), { 'reveal_sensitive_prefix': '0'}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_AUTH_TOKEN': auth_token}) resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(log_parts[9], '...') # Avoids pyflakes error, "local variable 'resp_body' is assigned to # but never used self.assertTrue(resp_body is not None) def test_ensure_fields(self): app = proxy_logging.ProxyLoggingMiddleware(FakeApp(), {}) app.access_logger = FakeLogger() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) with mock.patch('time.time', mock.MagicMock( side_effect=[10000000.0, 10000001.0])): resp = app(req.environ, start_response) resp_body = ''.join(resp) log_parts = self._log_parts(app) self.assertEquals(len(log_parts), 20) self.assertEquals(log_parts[0], '-') self.assertEquals(log_parts[1], '-') self.assertEquals(log_parts[2], '26/Apr/1970/17/46/41') self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(log_parts[7], '-') self.assertEquals(log_parts[8], '-') self.assertEquals(log_parts[9], '-') self.assertEquals(log_parts[10], '-') self.assertEquals(resp_body, 'FAKE APP') self.assertEquals(log_parts[11], str(len(resp_body))) self.assertEquals(log_parts[12], '-') self.assertEquals(log_parts[13], '-') self.assertEquals(log_parts[14], '-') self.assertEquals(log_parts[15], '1.0000') self.assertEquals(log_parts[16], '-') self.assertEquals(log_parts[17], '-') self.assertEquals(log_parts[18], '10000000.000000000') self.assertEquals(log_parts[19], '10000001.000000000') def test_dual_logging_middlewares(self): # Since no internal request is being made, outer most proxy logging # middleware, log1, should have performed the logging. app = FakeApp() flg0 = FakeLogger() env = {} log0 = proxy_logging.ProxyLoggingMiddleware(app, env, logger=flg0) flg1 = FakeLogger() log1 = proxy_logging.ProxyLoggingMiddleware(log0, env, logger=flg1) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = log1(req.environ, start_response) resp_body = ''.join(resp) self._log_parts(log0, should_be_empty=True) log_parts = self._log_parts(log1) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'FAKE APP') self.assertEquals(log_parts[11], str(len(resp_body))) def test_dual_logging_middlewares_w_inner(self): class FakeMiddleware(object): """ Fake middleware to make a separate internal request, but construct the response with different data. """ def __init__(self, app, conf): self.app = app self.conf = conf def GET(self, req): # Make the internal request ireq = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(ireq.environ, start_response) resp_body = ''.join(resp) if resp_body != 'FAKE APP': return Response(request=req, body="FAKE APP WAS NOT RETURNED", content_type="text/plain") # But our response is different return Response(request=req, body="FAKE MIDDLEWARE", content_type="text/plain") def __call__(self, env, start_response): req = Request(env) return self.GET(req)(env, start_response) # Since an internal request is being made, inner most proxy logging # middleware, log0, should have performed the logging. app = FakeApp() flg0 = FakeLogger() env = {} log0 = proxy_logging.ProxyLoggingMiddleware(app, env, logger=flg0) fake = FakeMiddleware(log0, env) flg1 = FakeLogger() log1 = proxy_logging.ProxyLoggingMiddleware(fake, env, logger=flg1) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = log1(req.environ, start_response) resp_body = ''.join(resp) # Inner most logger should have logged the app's response log_parts = self._log_parts(log0) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(log_parts[11], str(len('FAKE APP'))) # Outer most logger should have logged the other middleware's response log_parts = self._log_parts(log1) self.assertEquals(log_parts[3], 'GET') self.assertEquals(log_parts[4], '/') self.assertEquals(log_parts[5], 'HTTP/1.0') self.assertEquals(log_parts[6], '200') self.assertEquals(resp_body, 'FAKE MIDDLEWARE') self.assertEquals(log_parts[11], str(len(resp_body))) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_keystoneauth.py0000664000175400017540000004221112323703611025564 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.middleware import keystoneauth from swift.common.swob import Request, Response from swift.common.http import HTTP_FORBIDDEN from test.unit import FakeLogger class FakeApp(object): def __init__(self, status_headers_body_iter=None): self.calls = 0 self.status_headers_body_iter = status_headers_body_iter if not self.status_headers_body_iter: self.status_headers_body_iter = iter([('404 Not Found', {}, '')]) def __call__(self, env, start_response): self.calls += 1 self.request = Request.blank('', environ=env) if 'swift.authorize' in env: resp = env['swift.authorize'](self.request) if resp: return resp(env, start_response) status, headers, body = self.status_headers_body_iter.next() return Response(status=status, headers=headers, body=body)(env, start_response) class SwiftAuth(unittest.TestCase): def setUp(self): self.test_auth = keystoneauth.filter_factory({})(FakeApp()) self.test_auth.logger = FakeLogger() def _make_request(self, path=None, headers=None, **kwargs): if not path: path = '/v1/%s/c/o' % self.test_auth._get_account_for_tenant('foo') return Request.blank(path, headers=headers, **kwargs) def _get_identity_headers(self, status='Confirmed', tenant_id='1', tenant_name='acct', user='usr', role=''): return dict(X_IDENTITY_STATUS=status, X_TENANT_ID=tenant_id, X_TENANT_NAME=tenant_name, X_ROLES=role, X_USER_NAME=user) def _get_successful_middleware(self): response_iter = iter([('200 OK', {}, '')]) return keystoneauth.filter_factory({})(FakeApp(response_iter)) def test_invalid_request_authorized(self): role = self.test_auth.reseller_admin_role headers = self._get_identity_headers(role=role) req = self._make_request('/', headers=headers) resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 404) def test_invalid_request_non_authorized(self): req = self._make_request('/') resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 404) def test_confirmed_identity_is_authorized(self): role = self.test_auth.reseller_admin_role headers = self._get_identity_headers(role=role) req = self._make_request('/v1/AUTH_acct/c', headers) resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 200) def test_detect_reseller_request(self): role = self.test_auth.reseller_admin_role headers = self._get_identity_headers(role=role) req = self._make_request('/v1/AUTH_acct/c', headers) req.get_response(self._get_successful_middleware()) self.assertTrue(req.environ.get('reseller_request')) def test_confirmed_identity_is_not_authorized(self): headers = self._get_identity_headers() req = self._make_request('/v1/AUTH_acct/c', headers) resp = req.get_response(self.test_auth) self.assertEqual(resp.status_int, 403) def test_anonymous_is_authorized_for_permitted_referrer(self): req = self._make_request(headers={'X_IDENTITY_STATUS': 'Invalid'}) req.acl = '.r:*' resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 200) def test_anonymous_with_validtoken_authorized_for_permitted_referrer(self): req = self._make_request(headers={'X_IDENTITY_STATUS': 'Confirmed'}) req.acl = '.r:*' resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 200) def test_anonymous_is_not_authorized_for_unknown_reseller_prefix(self): req = self._make_request(path='/v1/BLAH_foo/c/o', headers={'X_IDENTITY_STATUS': 'Invalid'}) resp = req.get_response(self.test_auth) self.assertEqual(resp.status_int, 401) def test_blank_reseller_prefix(self): conf = {'reseller_prefix': ''} test_auth = keystoneauth.filter_factory(conf)(FakeApp()) account = tenant_id = 'foo' self.assertTrue(test_auth._reseller_check(account, tenant_id)) def test_reseller_prefix_added_underscore(self): conf = {'reseller_prefix': 'AUTH'} test_auth = keystoneauth.filter_factory(conf)(FakeApp()) self.assertEqual(test_auth.reseller_prefix, "AUTH_") def test_reseller_prefix_not_added_double_underscores(self): conf = {'reseller_prefix': 'AUTH_'} test_auth = keystoneauth.filter_factory(conf)(FakeApp()) self.assertEqual(test_auth.reseller_prefix, "AUTH_") def test_override_asked_for_but_not_allowed(self): conf = {'allow_overrides': 'false'} self.test_auth = keystoneauth.filter_factory(conf)(FakeApp()) req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 401) def test_override_asked_for_and_allowed(self): conf = {'allow_overrides': 'true'} self.test_auth = keystoneauth.filter_factory(conf)(FakeApp()) req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 404) def test_override_default_allowed(self): req = self._make_request('/v1/AUTH_account', environ={'swift.authorize_override': True}) resp = req.get_response(self.test_auth) self.assertEquals(resp.status_int, 404) def test_anonymous_options_allowed(self): req = self._make_request('/v1/AUTH_account', environ={'REQUEST_METHOD': 'OPTIONS'}) resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 200) def test_identified_options_allowed(self): headers = self._get_identity_headers() headers['REQUEST_METHOD'] = 'OPTIONS' req = self._make_request('/v1/AUTH_account', headers=self._get_identity_headers(), environ={'REQUEST_METHOD': 'OPTIONS'}) resp = req.get_response(self._get_successful_middleware()) self.assertEqual(resp.status_int, 200) def test_auth_scheme(self): req = self._make_request(path='/v1/BLAH_foo/c/o', headers={'X_IDENTITY_STATUS': 'Invalid'}) resp = req.get_response(self.test_auth) self.assertEqual(resp.status_int, 401) self.assertTrue('Www-Authenticate' in resp.headers) class TestAuthorize(unittest.TestCase): def setUp(self): self.test_auth = keystoneauth.filter_factory({})(FakeApp()) self.test_auth.logger = FakeLogger() def _make_request(self, path, **kwargs): return Request.blank(path, **kwargs) def _get_account(self, identity=None): if not identity: identity = self._get_identity() return self.test_auth._get_account_for_tenant( identity['HTTP_X_TENANT_ID']) def _get_identity(self, tenant_id='tenant_id', tenant_name='tenant_name', user_id='user_id', user_name='user_name', roles=[]): if isinstance(roles, list): roles = ','.join(roles) return {'HTTP_X_USER_ID': user_id, 'HTTP_X_USER_NAME': user_name, 'HTTP_X_TENANT_ID': tenant_id, 'HTTP_X_TENANT_NAME': tenant_name, 'HTTP_X_ROLES': roles, 'HTTP_X_IDENTITY_STATUS': 'Confirmed'} def _check_authenticate(self, account=None, identity=None, headers=None, exception=None, acl=None, env=None, path=None): if not identity: identity = self._get_identity() if not account: account = self._get_account(identity) if not path: path = '/v1/%s/c' % account default_env = {'REMOTE_USER': identity['HTTP_X_TENANT_ID']} default_env.update(identity) if env: default_env.update(env) req = self._make_request(path, headers=headers, environ=default_env) req.acl = acl result = self.test_auth.authorize(req) # if we have requested an exception but nothing came back then if exception and not result: self.fail("error %s was not returned" % (str(exception))) elif exception: self.assertEquals(result.status_int, exception) else: self.assertTrue(result is None) return req def test_authorize_fails_for_unauthorized_user(self): self._check_authenticate(exception=HTTP_FORBIDDEN) def test_authorize_fails_for_invalid_reseller_prefix(self): self._check_authenticate(account='BLAN_a', exception=HTTP_FORBIDDEN) def test_authorize_succeeds_for_reseller_admin(self): roles = [self.test_auth.reseller_admin_role] identity = self._get_identity(roles=roles) req = self._check_authenticate(identity=identity) self.assertTrue(req.environ.get('swift_owner')) def test_authorize_succeeds_for_insensitive_reseller_admin(self): roles = [self.test_auth.reseller_admin_role.upper()] identity = self._get_identity(roles=roles) req = self._check_authenticate(identity=identity) self.assertTrue(req.environ.get('swift_owner')) def test_authorize_succeeds_as_owner_for_operator_role(self): roles = self.test_auth.operator_roles.split(',') identity = self._get_identity(roles=roles) req = self._check_authenticate(identity=identity) self.assertTrue(req.environ.get('swift_owner')) def test_authorize_succeeds_as_owner_for_insensitive_operator_role(self): roles = [r.upper() for r in self.test_auth.operator_roles.split(',')] identity = self._get_identity(roles=roles) req = self._check_authenticate(identity=identity) self.assertTrue(req.environ.get('swift_owner')) def _check_authorize_for_tenant_owner_match(self, exception=None): identity = self._get_identity(user_name='same_name', tenant_name='same_name') req = self._check_authenticate(identity=identity, exception=exception) expected = bool(exception is None) self.assertEqual(bool(req.environ.get('swift_owner')), expected) def test_authorize_succeeds_as_owner_for_tenant_owner_match(self): self.test_auth.is_admin = True self._check_authorize_for_tenant_owner_match() def test_authorize_fails_as_owner_for_tenant_owner_match(self): self.test_auth.is_admin = False self._check_authorize_for_tenant_owner_match( exception=HTTP_FORBIDDEN) def test_authorize_succeeds_for_container_sync(self): env = {'swift_sync_key': 'foo', 'REMOTE_ADDR': '127.0.0.1'} headers = {'x-container-sync-key': 'foo', 'x-timestamp': '1'} self._check_authenticate(env=env, headers=headers) def test_authorize_fails_for_invalid_referrer(self): env = {'HTTP_REFERER': 'http://invalid.com/index.html'} self._check_authenticate(acl='.r:example.com', env=env, exception=HTTP_FORBIDDEN) def test_authorize_fails_for_referrer_without_rlistings(self): env = {'HTTP_REFERER': 'http://example.com/index.html'} self._check_authenticate(acl='.r:example.com', env=env, exception=HTTP_FORBIDDEN) def test_authorize_succeeds_for_referrer_with_rlistings(self): env = {'HTTP_REFERER': 'http://example.com/index.html'} self._check_authenticate(acl='.r:example.com,.rlistings', env=env) def test_authorize_succeeds_for_referrer_with_obj(self): path = '/v1/%s/c/o' % self._get_account() env = {'HTTP_REFERER': 'http://example.com/index.html'} self._check_authenticate(acl='.r:example.com', env=env, path=path) def test_authorize_succeeds_for_user_role_in_roles(self): acl = 'allowme' identity = self._get_identity(roles=[acl]) self._check_authenticate(identity=identity, acl=acl) def test_authorize_succeeds_for_tenant_name_user_in_roles(self): identity = self._get_identity() user_name = identity['HTTP_X_USER_NAME'] user_id = identity['HTTP_X_USER_ID'] tenant_id = identity['HTTP_X_TENANT_ID'] for user in [user_id, user_name, '*']: acl = '%s:%s' % (tenant_id, user) self._check_authenticate(identity=identity, acl=acl) def test_authorize_succeeds_for_tenant_id_user_in_roles(self): identity = self._get_identity() user_name = identity['HTTP_X_USER_NAME'] user_id = identity['HTTP_X_USER_ID'] tenant_name = identity['HTTP_X_TENANT_NAME'] for user in [user_id, user_name, '*']: acl = '%s:%s' % (tenant_name, user) self._check_authenticate(identity=identity, acl=acl) def test_authorize_succeeds_for_wildcard_tenant_user_in_roles(self): identity = self._get_identity() user_name = identity['HTTP_X_USER_NAME'] user_id = identity['HTTP_X_USER_ID'] for user in [user_id, user_name, '*']: acl = '*:%s' % user self._check_authenticate(identity=identity, acl=acl) def test_cross_tenant_authorization_success(self): self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantID:userA']), 'tenantID:userA') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantNAME:userA']), 'tenantNAME:userA') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['*:userA']), '*:userA') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantID:userID']), 'tenantID:userID') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantNAME:userID']), 'tenantNAME:userID') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['*:userID']), '*:userID') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantID:*']), 'tenantID:*') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantNAME:*']), 'tenantNAME:*') self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['*:*']), '*:*') def test_cross_tenant_authorization_failure(self): self.assertEqual( self.test_auth._authorize_cross_tenant( 'userID', 'userA', 'tenantID', 'tenantNAME', ['tenantXYZ:userA']), None) def test_delete_own_account_not_allowed(self): roles = self.test_auth.operator_roles.split(',') identity = self._get_identity(roles=roles) account = self._get_account(identity) self._check_authenticate(account=account, identity=identity, exception=HTTP_FORBIDDEN, path='/v1/' + account, env={'REQUEST_METHOD': 'DELETE'}) def test_delete_own_account_when_reseller_allowed(self): roles = [self.test_auth.reseller_admin_role] identity = self._get_identity(roles=roles) account = self._get_account(identity) req = self._check_authenticate(account=account, identity=identity, path='/v1/' + account, env={'REQUEST_METHOD': 'DELETE'}) self.assertEqual(bool(req.environ.get('swift_owner')), True) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_acl.py0000664000175400017540000002260512323703611023605 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.middleware import acl class TestACL(unittest.TestCase): def test_clean_acl(self): value = acl.clean_acl('header', '.r:*') self.assertEquals(value, '.r:*') value = acl.clean_acl('header', '.r:specific.host') self.assertEquals(value, '.r:specific.host') value = acl.clean_acl('header', '.r:.ending.with') self.assertEquals(value, '.r:.ending.with') value = acl.clean_acl('header', '.r:*.ending.with') self.assertEquals(value, '.r:.ending.with') value = acl.clean_acl('header', '.r:-*.ending.with') self.assertEquals(value, '.r:-.ending.with') value = acl.clean_acl('header', '.r:one,.r:two') self.assertEquals(value, '.r:one,.r:two') value = acl.clean_acl('header', '.r:*,.r:-specific.host') self.assertEquals(value, '.r:*,.r:-specific.host') value = acl.clean_acl('header', '.r:*,.r:-.ending.with') self.assertEquals(value, '.r:*,.r:-.ending.with') value = acl.clean_acl('header', '.r:one,.r:-two') self.assertEquals(value, '.r:one,.r:-two') value = acl.clean_acl('header', '.r:one,.r:-two,account,account:user') self.assertEquals(value, '.r:one,.r:-two,account,account:user') value = acl.clean_acl('header', 'TEST_account') self.assertEquals(value, 'TEST_account') value = acl.clean_acl('header', '.ref:*') self.assertEquals(value, '.r:*') value = acl.clean_acl('header', '.referer:*') self.assertEquals(value, '.r:*') value = acl.clean_acl('header', '.referrer:*') self.assertEquals(value, '.r:*') value = acl.clean_acl('header', ' .r : one , ,, .r:two , .r : - three ') self.assertEquals(value, '.r:one,.r:two,.r:-three') self.assertRaises(ValueError, acl.clean_acl, 'header', '.unknown:test') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:*.') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : * . ') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-*.') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : - * . ') self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : ') self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : ') self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-') self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : - ') self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : - ') self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.r:r') def test_parse_acl(self): self.assertEquals(acl.parse_acl(None), ([], [])) self.assertEquals(acl.parse_acl(''), ([], [])) self.assertEquals(acl.parse_acl('.r:ref1'), (['ref1'], [])) self.assertEquals(acl.parse_acl('.r:-ref1'), (['-ref1'], [])) self.assertEquals(acl.parse_acl('account:user'), ([], ['account:user'])) self.assertEquals(acl.parse_acl('account'), ([], ['account'])) self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.r:ref3,.r:-ref4'), (['ref3', '-ref4'], ['acc1', 'acc2:usr2'])) self.assertEquals(acl.parse_acl( 'acc1,acc2:usr2,.r:ref3,acc3,acc4:usr4,.r:ref5,.r:-ref6'), (['ref3', 'ref5', '-ref6'], ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4'])) def test_parse_v2_acl(self): # For all these tests, the header name will be "hdr". tests = [ # Simple case: all ACL data in one header line ({'hdr': '{"a":1,"b":"foo"}'}, {'a': 1, 'b': 'foo'}), # No header "hdr" exists -- should return None ({}, None), ({'junk': 'junk'}, None), # Empty ACLs should return empty dict ({'hdr': ''}, {}), ({'hdr': '{}'}, {}), ({'hdr': '{ }'}, {}), # Bad input -- should return None ({'hdr': '["array"]'}, None), ({'hdr': 'null'}, None), ({'hdr': '"some_string"'}, None), ({'hdr': '123'}, None), ] for hdrs_in, expected in tests: result = acl.parse_acl(version=2, data=hdrs_in.get('hdr')) self.assertEquals(expected, result, '%r: %r != %r' % (hdrs_in, result, expected)) def test_format_v1_acl(self): tests = [ ((['a', 'b'], ['c.com']), 'a,b,.r:c.com'), ((['a', 'b'], ['c.com', '-x.c.com']), 'a,b,.r:c.com,.r:-x.c.com'), ((['a', 'b'], None), 'a,b'), ((None, ['c.com']), '.r:c.com'), ((None, None), ''), ] for (groups, refs), expected in tests: result = acl.format_acl( version=1, groups=groups, referrers=refs, header_name='hdr') self.assertEquals(expected, result, 'groups=%r, refs=%r: %r != %r' % (groups, refs, result, expected)) def test_format_v2_acl(self): tests = [ ({}, '{}'), ({'foo': 'bar'}, '{"foo":"bar"}'), ({'groups': ['a', 'b'], 'referrers': ['c.com', '-x.c.com']}, '{"groups":["a","b"],"referrers":["c.com","-x.c.com"]}'), ] for data, expected in tests: result = acl.format_acl(version=2, acl_dict=data) self.assertEquals(expected, result, 'data=%r: %r *!=* %r' % (data, result, expected)) def test_acls_from_account_info(self): test_data = [ ({}, None), ({'sysmeta': {}}, None), ({'sysmeta': {'core-access-control': '{"VERSION":1,"admin":["a","b"]}'}}, {'admin': ['a', 'b'], 'read-write': [], 'read-only': []}), ({ 'some-key': 'some-value', 'other-key': 'other-value', 'sysmeta': { 'core-access-control': '{"VERSION":1,"admin":["a","b"],"r' 'ead-write":["c"],"read-only":[]}', }}, {'admin': ['a', 'b'], 'read-write': ['c'], 'read-only': []}), ] for args, expected in test_data: result = acl.acls_from_account_info(args) self.assertEqual(expected, result, "%r: Got %r, expected %r" % (args, result, expected)) def test_referrer_allowed(self): self.assert_(not acl.referrer_allowed('host', None)) self.assert_(not acl.referrer_allowed('host', [])) self.assert_(acl.referrer_allowed(None, ['*'])) self.assert_(acl.referrer_allowed('', ['*'])) self.assert_(not acl.referrer_allowed(None, ['specific.host'])) self.assert_(not acl.referrer_allowed('', ['specific.host'])) self.assert_(acl.referrer_allowed('http://www.example.com/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://user@www.example.com/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://user:pass@www.example.com/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://www.example.com:8080/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://user@www.example.com:8080/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://user:pass@www.example.com:8080/index.html', ['.example.com'])) self.assert_(acl.referrer_allowed( 'http://user:pass@www.example.com:8080', ['.example.com'])) self.assert_(acl.referrer_allowed('http://www.example.com', ['.example.com'])) self.assert_(not acl.referrer_allowed( 'http://thief.example.com', ['.example.com', '-thief.example.com'])) self.assert_(not acl.referrer_allowed( 'http://thief.example.com', ['*', '-thief.example.com'])) self.assert_(acl.referrer_allowed( 'http://www.example.com', ['.other.com', 'www.example.com'])) self.assert_(acl.referrer_allowed( 'http://www.example.com', ['-.example.com', 'www.example.com'])) # This is considered a relative uri to the request uri, a mode not # currently supported. self.assert_(not acl.referrer_allowed('www.example.com', ['.example.com'])) self.assert_(not acl.referrer_allowed('../index.html', ['.example.com'])) self.assert_(acl.referrer_allowed('www.example.com', ['*'])) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_ratelimit.py0000664000175400017540000005205612323703611025043 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time import eventlet import mock from contextlib import contextmanager from threading import Thread from test.unit import FakeLogger from swift.common.middleware import ratelimit from swift.proxy.controllers.base import get_container_memcache_key, \ headers_to_container_info from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request class FakeMemcache(object): def __init__(self): self.store = {} self.error_on_incr = False self.init_incr_return_neg = False def get(self, key): return self.store.get(key) def set(self, key, value, serialize=False, time=0): self.store[key] = value return True def incr(self, key, delta=1, time=0): if self.error_on_incr: raise MemcacheConnectionError('Memcache restarting') if self.init_incr_return_neg: # simulate initial hit, force reset of memcache self.init_incr_return_neg = False return -10000000 self.store[key] = int(self.store.setdefault(key, 0)) + int(delta) if self.store[key] < 0: self.store[key] = 0 return int(self.store[key]) def decr(self, key, delta=1, time=0): return self.incr(key, delta=-delta, time=time) @contextmanager def soft_lock(self, key, timeout=0, retries=5): yield True def delete(self, key): try: del self.store[key] except Exception: pass return True def mock_http_connect(response, headers=None, with_exc=False): class FakeConn(object): def __init__(self, status, headers, with_exc): self.status = status self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' self.with_exc = with_exc self.headers = headers if self.headers is None: self.headers = {} def getresponse(self): if self.with_exc: raise Exception('test') return self def getheader(self, header): return self.headers[header] def read(self, amt=None): return '' def close(self): return return lambda *args, **kwargs: FakeConn(response, headers, with_exc) class FakeApp(object): def __call__(self, env, start_response): return ['204 No Content'] def start_response(*args): pass def dummy_filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def limit_filter(app): return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger()) return limit_filter time_ticker = 0 time_override = [] def mock_sleep(x): global time_ticker time_ticker += x def mock_time(): global time_override global time_ticker if time_override: cur_time = time_override.pop(0) if cur_time is None: time_override = [None if i is None else i + time_ticker for i in time_override] return time_ticker return cur_time return time_ticker class TestRateLimit(unittest.TestCase): def _reset_time(self): global time_ticker time_ticker = 0 def setUp(self): self.was_sleep = eventlet.sleep eventlet.sleep = mock_sleep self.was_time = time.time time.time = mock_time self._reset_time() def tearDown(self): eventlet.sleep = self.was_sleep time.time = self.was_time def _run(self, callable_func, num, rate, check_time=True): global time_ticker begin = time.time() for x in range(num): callable_func() end = time.time() total_time = float(num) / rate - 1.0 / rate # 1st request not limited # Allow for one second of variation in the total time. time_diff = abs(total_time - (end - begin)) if check_time: self.assertEquals(round(total_time, 1), round(time_ticker, 1)) return time_diff def test_get_maxrate(self): conf_dict = {'container_ratelimit_10': 200, 'container_ratelimit_50': 100, 'container_ratelimit_75': 30} test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 0), None) self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 5), None) self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 10), 200) self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 60), 72) self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 160), 30) def test_get_ratelimitable_key_tuples(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'object_count': '5'} the_app = ratelimit.RateLimitMiddleware(None, conf_dict, logger=FakeLogger()) the_app.memcache_client = fake_memcache req = lambda: None req.environ = {} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): req.method = 'DELETE' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', None, None)), 0) req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1) req.method = 'DELETE' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1) req.method = 'GET' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', 'o')), 0) req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', 'o')), 1) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 10}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 2) self.assertEquals(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10)) def get_fake_ratelimit(*args, **kwargs): return {'sysmeta': {'global-write-ratelimit': 'notafloat'}} with mock.patch('swift.common.middleware.ratelimit.get_account_info', get_fake_ratelimit): req.method = 'PUT' self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', 'c', None)), 1) def test_memcached_container_info_dict(self): mdict = headers_to_container_info({'x-container-object-count': '45'}) self.assertEquals(mdict['object_count'], '45') def test_ratelimit_old_memcache_format(self): current_rate = 13 conf_dict = {'account_ratelimit': current_rate, 'container_ratelimit_3': 200} fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'container_size': 5} the_app = ratelimit.RateLimitMiddleware(None, conf_dict, logger=FakeLogger()) the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' req.environ = {} with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o') self.assertEquals(tuples, [('ratelimit/a/c', 200.0)]) def test_account_ratelimit(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): for meth, exp_time in [ ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: req = Request.blank('/v/a%s/c' % meth) req.method = meth req.environ['swift.cache'] = FakeMemcache() make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=bool(exp_time)) self.assertEquals(round(time.time() - begin, 1), exp_time) self._reset_time() def test_ratelimit_set_incr(self): current_rate = 5 num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].init_incr_return_neg = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) self.assertEquals(round(time.time() - begin, 1), 9.8) def test_ratelimit_whitelist(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_498s = [ t for t in threads if ''.join(t.result).startswith('Slow down')] self.assertEquals(len(the_498s), 0) self.assertEquals(time_ticker, 0) def test_ratelimit_blacklist(self): global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) self.test_ratelimit.BLACK_LIST_SLEEP = 0 ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/b/c') req.environ['swift.cache'] = FakeMemcache() class rate_caller(Thread): def __init__(self, parent): Thread.__init__(self) self.parent = parent def run(self): self.result = self.parent.test_ratelimit(req.environ, start_response) nt = 5 threads = [] for i in range(nt): rc = rate_caller(self) rc.start() threads.append(rc) for thread in threads: thread.join() the_497s = [ t for t in threads if ''.join(t.result).startswith('Your account')] self.assertEquals(len(the_497s), 5) self.assertEquals(time_ticker, 0) def test_ratelimit_max_rate_double(self): global time_ticker global time_override current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_double_container(self): global time_ticker global time_override current_rate = 2 conf_dict = {'container_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c/o') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( ratelimit.get_container_memcache_key('a', 'c'), {'container_size': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_double_container_listing(self): global time_ticker global time_override current_rate = 2 conf_dict = {'container_listing_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') req.method = 'GET' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].set( ratelimit.get_container_memcache_key('a', 'c'), {'container_size': 1}) time_override = [0, 0, 0, 0, None] # simulates 4 requests coming in at same time, then sleeping r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down') mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_multiple_acc(self): num_calls = 4 current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2} fake_memcache = FakeMemcache() the_app = ratelimit.RateLimitMiddleware(None, conf_dict, logger=FakeLogger()) the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' req.environ = {} class rate_caller(Thread): def __init__(self, name): self.myname = name Thread.__init__(self) def run(self): for j in range(num_calls): self.result = the_app.handle_ratelimit(req, self.myname, 'c', None) with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): nt = 15 begin = time.time() threads = [] for i in range(nt): rc = rate_caller('a%s' % i) rc.start() threads.append(rc) for thread in threads: thread.join() time_took = time.time() - begin self.assertEquals(1.5, round(time_took, 1)) def test_call_invalid_path(self): env = {'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '//v1/AUTH_1234567890', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '80', 'swift.cache': FakeMemcache(), 'SERVER_PROTOCOL': 'HTTP/1.0'} app = lambda *args, **kwargs: ['fake_app'] rate_mid = ratelimit.RateLimitMiddleware(app, {}, logger=FakeLogger()) class a_callable(object): def __call__(self, *args, **kwargs): pass resp = rate_mid.__call__(env, a_callable()) self.assert_('fake_app' == resp[0]) def test_no_memcache(self): current_rate = 13 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a') req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting def test_restarting_memcache(self): current_rate = 2 num_calls = 5 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.method = 'PUT' req.environ['swift.cache'] = FakeMemcache() req.environ['swift.cache'].error_on_incr = True make_app_call = lambda: self.test_ratelimit(req.environ, start_response) begin = time.time() with mock.patch('swift.common.middleware.ratelimit.get_account_info', lambda *args, **kwargs: {}): self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin self.assertEquals(round(time_took, 1), 0) # no memcache, no limit if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_list_endpoints.py0000664000175400017540000001736412323703611026112 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import array import unittest from tempfile import mkdtemp from shutil import rmtree import os from swift.common import ring, utils from swift.common.utils import json from swift.common.swob import Request, Response from swift.common.middleware import list_endpoints class FakeApp(object): def __call__(self, env, start_response): return Response(body="FakeApp")(env, start_response) def start_response(*args): pass class TestListEndpoints(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' self.testdir = mkdtemp() accountgz = os.path.join(self.testdir, 'account.ring.gz') containergz = os.path.join(self.testdir, 'container.ring.gz') objectgz = os.path.join(self.testdir, 'object.ring.gz') # Let's make the rings slightly different so we can test # that the correct ring is consulted (e.g. we don't consult # the object ring to get nodes for a container) intended_replica2part2dev_id_a = [ array.array('H', [3, 1, 3, 1]), array.array('H', [0, 3, 1, 4]), array.array('H', [1, 4, 0, 3])] intended_replica2part2dev_id_c = [ array.array('H', [4, 3, 0, 1]), array.array('H', [0, 1, 3, 4]), array.array('H', [3, 4, 0, 1])] intended_replica2part2dev_id_o = [ array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]), array.array('H', [3, 4, 3, 4])] intended_devs = [{'id': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'device': 'sda1'}, {'id': 1, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'device': 'sdb1'}, None, {'id': 3, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000, 'device': 'sdc1'}, {'id': 4, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000, 'device': 'sdd1'}] intended_part_shift = 30 ring.RingData(intended_replica2part2dev_id_a, intended_devs, intended_part_shift).save(accountgz) ring.RingData(intended_replica2part2dev_id_c, intended_devs, intended_part_shift).save(containergz) ring.RingData(intended_replica2part2dev_id_o, intended_devs, intended_part_shift).save(objectgz) self.app = FakeApp() self.list_endpoints = list_endpoints.filter_factory( {'swift_dir': self.testdir})(self.app) def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_get_endpoint(self): # Expected results for objects taken from test_ring # Expected results for others computed by manually invoking # ring.get_nodes(). resp = Request.blank('/endpoints/a/c/o1').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) # Here, 'o1/' is the object name. resp = Request.blank('/endpoints/a/c/o1/').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c/o1/", "http://10.1.2.2:6000/sdd1/3/a/c/o1/" ]) resp = Request.blank('/endpoints/a/c2').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sda1/2/a/c2", "http://10.1.2.1:6000/sdc1/2/a/c2" ]) resp = Request.blank('/endpoints/a1').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.2.1:6000/sdc1/0/a1", "http://10.1.1.1:6000/sda1/0/a1", "http://10.1.1.1:6000/sdb1/0/a1" ]) resp = Request.blank('/endpoints/').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 400) resp = Request.blank('/endpoints/a/c 2').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c%202", "http://10.1.2.2:6000/sdd1/3/a/c%202" ]) resp = Request.blank('/endpoints/a/c%202').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/a/c%202", "http://10.1.2.2:6000/sdd1/3/a/c%202" ]) resp = Request.blank('/endpoints/ac%20count/con%20tainer/ob%20ject') \ .get_response(self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/3/ac%20count/con%20tainer/ob%20ject", "http://10.1.2.2:6000/sdd1/3/ac%20count/con%20tainer/ob%20ject" ]) resp = Request.blank('/endpoints/a/c/o1', {'REQUEST_METHOD': 'POST'}) \ .get_response(self.list_endpoints) self.assertEquals(resp.status_int, 405) self.assertEquals(resp.status, '405 Method Not Allowed') self.assertEquals(resp.headers['allow'], 'GET') resp = Request.blank('/not-endpoints').get_response( self.list_endpoints) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.status, '200 OK') self.assertEquals(resp.body, 'FakeApp') # test custom path with trailing slash custom_path_le = list_endpoints.filter_factory({ 'swift_dir': self.testdir, 'list_endpoints_path': '/some/another/path/' })(self.app) resp = Request.blank('/some/another/path/a/c/o1') \ .get_response(custom_path_le) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) # test ustom path without trailing slash custom_path_le = list_endpoints.filter_factory({ 'swift_dir': self.testdir, 'list_endpoints_path': '/some/another/path' })(self.app) resp = Request.blank('/some/another/path/a/c/o1') \ .get_response(custom_path_le) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(json.loads(resp.body), [ "http://10.1.1.1:6000/sdb1/1/a/c/o1", "http://10.1.2.2:6000/sdd1/1/a/c/o1" ]) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_memcache.py0000664000175400017540000002702012323703611024604 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from ConfigParser import NoSectionError, NoOptionError from swift.common.middleware import memcache from swift.common.memcached import MemcacheRing from swift.common.swob import Request class FakeApp(object): def __call__(self, env, start_response): return env class ExcConfigParser(object): def read(self, path): raise Exception('read called with %r' % path) class EmptyConfigParser(object): def read(self, path): return False def get_config_parser(memcache_servers='1.2.3.4:5', memcache_serialization_support='1', memcache_max_connections='4', section='memcache'): _srvs = memcache_servers _sers = memcache_serialization_support _maxc = memcache_max_connections _section = section class SetConfigParser(object): def read(self, path): return True def get(self, section, option): if _section == section: if option == 'memcache_servers': if _srvs == 'error': raise NoOptionError(option, section) return _srvs elif option == 'memcache_serialization_support': if _sers == 'error': raise NoOptionError(option, section) return _sers elif option in ('memcache_max_connections', 'max_connections'): if _maxc == 'error': raise NoOptionError(option, section) return _maxc else: raise NoOptionError(option, section) else: raise NoSectionError(option) return SetConfigParser def start_response(*args): pass class TestCacheMiddleware(unittest.TestCase): def setUp(self): self.app = memcache.MemcacheMiddleware(FakeApp(), {}) def test_cache_middleware(self): req = Request.blank('/something', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertTrue('swift.cache' in resp) self.assertTrue(isinstance(resp['swift.cache'], MemcacheRing)) def test_conf_default_read(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = ExcConfigParser count = 0 try: for d in ({}, {'memcache_servers': '6.7.8.9:10'}, {'memcache_serialization_support': '0'}, {'memcache_max_connections': '30'}, {'memcache_servers': '6.7.8.9:10', 'memcache_serialization_support': '0'}, {'memcache_servers': '6.7.8.9:10', 'memcache_max_connections': '30'}, {'memcache_serialization_support': '0', 'memcache_max_connections': '30'} ): try: memcache.MemcacheMiddleware(FakeApp(), d) except Exception as err: self.assertEquals( str(err), "read called with '/etc/swift/memcache.conf'") count += 1 finally: memcache.ConfigParser = orig_parser self.assertEquals(count, 7) def test_conf_set_no_read(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = ExcConfigParser exc = None try: memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '1.2.3.4:5', 'memcache_serialization_support': '2', 'memcache_max_connections': '30'}) except Exception as err: exc = err finally: memcache.ConfigParser = orig_parser self.assertEquals(exc, None) def test_conf_default(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = EmptyConfigParser try: app = memcache.MemcacheMiddleware(FakeApp(), {}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '127.0.0.1:11211') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, False) self.assertEquals( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) def test_conf_inline(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', 'memcache_serialization_support': '0', 'memcache_max_connections': '5'}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '6.7.8.9:10') self.assertEquals(app.memcache._allow_pickle, True) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['6.7.8.9:10'].max_size, 5) def test_conf_extra_no_section(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser(section='foobar') try: app = memcache.MemcacheMiddleware(FakeApp(), {}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '127.0.0.1:11211') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, False) self.assertEquals( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) def test_conf_extra_no_option(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser( memcache_servers='error', memcache_serialization_support='error', memcache_max_connections='error') try: app = memcache.MemcacheMiddleware(FakeApp(), {}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '127.0.0.1:11211') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, False) self.assertEquals( app.memcache._client_cache['127.0.0.1:11211'].max_size, 2) def test_conf_inline_other_max_conn(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', 'memcache_serialization_support': '0', 'max_connections': '5'}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '6.7.8.9:10') self.assertEquals(app.memcache._allow_pickle, True) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['6.7.8.9:10'].max_size, 5) def test_conf_inline_bad_max_conn(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', 'memcache_serialization_support': '0', 'max_connections': 'bad42'}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '6.7.8.9:10') self.assertEquals(app.memcache._allow_pickle, True) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['6.7.8.9:10'].max_size, 4) def test_conf_from_extra_conf(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware(FakeApp(), {}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '1.2.3.4:5') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['1.2.3.4:5'].max_size, 4) def test_conf_from_extra_conf_bad_max_conn(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser( memcache_max_connections='bad42') try: app = memcache.MemcacheMiddleware(FakeApp(), {}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '1.2.3.4:5') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['1.2.3.4:5'].max_size, 2) def test_conf_from_inline_and_maxc_from_extra_conf(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', 'memcache_serialization_support': '0'}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '6.7.8.9:10') self.assertEquals(app.memcache._allow_pickle, True) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['6.7.8.9:10'].max_size, 4) def test_conf_from_inline_and_sers_from_extra_conf(self): orig_parser = memcache.ConfigParser memcache.ConfigParser = get_config_parser() try: app = memcache.MemcacheMiddleware( FakeApp(), {'memcache_servers': '6.7.8.9:10', 'memcache_max_connections': '42'}) finally: memcache.ConfigParser = orig_parser self.assertEquals(app.memcache_servers, '6.7.8.9:10') self.assertEquals(app.memcache._allow_pickle, False) self.assertEquals(app.memcache._allow_unpickle, True) self.assertEquals( app.memcache._client_cache['6.7.8.9:10'].max_size, 42) def test_filter_factory(self): factory = memcache.filter_factory({'max_connections': '3'}, memcache_servers='10.10.10.10:10', memcache_serialization_support='1') thefilter = factory('myapp') self.assertEquals(thefilter.app, 'myapp') self.assertEquals(thefilter.memcache_servers, '10.10.10.10:10') self.assertEquals(thefilter.memcache._allow_pickle, False) self.assertEquals(thefilter.memcache._allow_unpickle, True) self.assertEquals( thefilter.memcache._client_cache['10.10.10.10:10'].max_size, 3) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_domain_remap.py0000664000175400017540000001366112323703611025503 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request from swift.common.middleware import domain_remap class FakeApp(object): def __call__(self, env, start_response): return env['PATH_INFO'] def start_response(*args): pass class TestDomainRemap(unittest.TestCase): def setUp(self): self.app = domain_remap.DomainRemapMiddleware(FakeApp(), {}) def test_domain_remap_passthrough(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'SERVER_NAME': 'example.com'}, headers={'Host': None}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'example.com:8080'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/') def test_domain_remap_account(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'SERVER_NAME': 'AUTH_a.example.com'}, headers={'Host': None}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'AUTH-uuid.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_uuid') def test_domain_remap_account_container(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a/c') def test_domain_remap_extra_subdomains(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'x.y.c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, ['Bad domain in host header']) def test_domain_remap_account_with_path_root(self): req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a') def test_domain_remap_account_container_with_path_root(self): req = Request.blank('/v1', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a/c') def test_domain_remap_account_container_with_path(self): req = Request.blank('/obj', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a/c/obj') def test_domain_remap_account_container_with_path_root_and_path(self): req = Request.blank('/v1/obj', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/AUTH_a/c/obj') def test_domain_remap_account_matching_ending_not_domain(self): req = Request.blank('/dontchange', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.aexample.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/dontchange') def test_domain_remap_configured_with_empty_storage_domain(self): self.app = domain_remap.DomainRemapMiddleware(FakeApp(), {'storage_domain': ''}) req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.AUTH_a.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/test') def test_domain_remap_configured_with_prefixes(self): conf = {'reseller_prefixes': 'PREFIX'} self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf) req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.prefix_uuid.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/v1/PREFIX_uuid/c/test') def test_domain_remap_configured_with_bad_prefixes(self): conf = {'reseller_prefixes': 'UNKNOWN'} self.app = domain_remap.DomainRemapMiddleware(FakeApp(), conf) req = Request.blank('/test', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.prefix_uuid.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, '/test') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_account_quotas.py0000664000175400017540000004372612323703611026105 0ustar jenkinsjenkins00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request from swift.common.middleware import account_quotas from swift.proxy.controllers.base import _get_cache_key, \ headers_to_account_info, get_object_env_key, \ headers_to_object_info class FakeCache(object): def __init__(self, val): self.val = val def get(self, *args): return self.val def set(self, *args, **kwargs): pass class FakeBadApp(object): def __init__(self, headers=[]): self.headers = headers def __call__(self, env, start_response): start_response('404 NotFound', self.headers) return [] class FakeApp(object): def __init__(self, headers=[]): self.headers = headers def __call__(self, env, start_response): if env['REQUEST_METHOD'] == "HEAD" and \ env['PATH_INFO'] == '/v1/a/c2/o2': env_key = get_object_env_key('a', 'c2', 'o2') env[env_key] = headers_to_object_info(self.headers, 200) start_response('200 OK', self.headers) elif env['REQUEST_METHOD'] == "HEAD" and \ env['PATH_INFO'] == '/v1/a/c2/o3': start_response('404 Not Found', []) else: # Cache the account_info (same as a real application) cache_key, env_key = _get_cache_key('a', None) env[env_key] = headers_to_account_info(self.headers, 200) start_response('200 OK', self.headers) return [] class TestAccountQuota(unittest.TestCase): def test_unauthorized(self): headers = [('x-account-bytes-used', '1000'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) #Response code of 200 because authentication itself is not done here self.assertEquals(res.status_int, 200) def test_no_quotas(self): headers = [('x-account-bytes-used', '1000'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_obj_request_ignores_attempt_to_set_quotas(self): # If you try to set X-Account-Meta-* on an object, it's ignored, so # the quota middleware shouldn't complain about it even if we're not a # reseller admin. headers = [('x-account-bytes-used', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', headers={'X-Account-Meta-Quota-Bytes': '99999'}, environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_container_request_ignores_attempt_to_set_quotas(self): # As with an object, if you try to set X-Account-Meta-* on a # container, it's ignored. headers = [('x-account-bytes-used', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c', headers={'X-Account-Meta-Quota-Bytes': '99999'}, environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_bogus_quota_is_ignored(self): # This can happen if the metadata was set by a user prior to the # activation of the account-quota middleware headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', 'pasty-plastogene')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_bytes_quota(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', '0')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_over_quota_container_create_still_works(self): headers = [('x-account-bytes-used', '1001'), ('x-account-meta-quota-bytes', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/new_container', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_CONTAINER_META_BERT': 'ernie', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_over_quota_container_post_still_works(self): headers = [('x-account-bytes-used', '1001'), ('x-account-meta-quota-bytes', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/new_container', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_CONTAINER_META_BERT': 'ernie', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_over_quota_obj_post_still_works(self): headers = [('x-account-bytes-used', '1001'), ('x-account-meta-quota-bytes', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_OBJECT_META_BERT': 'ernie', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_bytes_quota_copy_from(self): headers = [('x-account-bytes-used', '500'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_exceed_bytes_quota_copy_verb(self): headers = [('x-account-bytes-used', '500'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 413) def test_not_exceed_bytes_quota_copy_from(self): headers = [('x-account-bytes-used', '0'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': '/c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_bytes_quota_copy_verb(self): headers = [('x-account-bytes-used', '0'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache}, headers={'Destination': '/c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_quota_copy_from_no_src(self): headers = [('x-account-bytes-used', '0'), ('x-account-meta-quota-bytes', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': '/c2/o3'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_quota_copy_from_bad_src(self): headers = [('x-account-bytes-used', '0'), ('x-account-meta-quota-bytes', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}, headers={'x-copy-from': 'bad_path'}) res = req.get_response(app) self.assertEquals(res.status_int, 412) def test_exceed_bytes_quota_reseller(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', '0')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'reseller_request': True}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_bytes_quota_reseller_copy_from(self): headers = [('x-account-bytes-used', '500'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache, 'reseller_request': True}, headers={'x-copy-from': 'c2/o2'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_exceed_bytes_quota_reseller_copy_verb(self): headers = [('x-account-bytes-used', '500'), ('x-account-meta-quota-bytes', '1000'), ('content-length', '1000')] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c2/o2', environ={'REQUEST_METHOD': 'COPY', 'swift.cache': cache, 'reseller_request': True}, headers={'Destination': 'c/o'}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_bad_application_quota(self): headers = [] app = account_quotas.AccountQuotaMiddleware(FakeBadApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 404) def test_no_info_quota(self): headers = [] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_not_exceed_bytes_quota(self): headers = [('x-account-bytes-used', '1000'), ('x-account-meta-quota-bytes', 2000)] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_invalid_quotas(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_ACCOUNT_META_QUOTA_BYTES': 'abc', 'reseller_request': True}) res = req.get_response(app) self.assertEquals(res.status_int, 400) def test_valid_quotas_admin(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_ACCOUNT_META_QUOTA_BYTES': '100'}) res = req.get_response(app) self.assertEquals(res.status_int, 403) def test_valid_quotas_reseller(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_ACCOUNT_META_QUOTA_BYTES': '100', 'reseller_request': True}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_delete_quotas(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_ACCOUNT_META_QUOTA_BYTES': ''}) res = req.get_response(app) self.assertEquals(res.status_int, 403) def test_delete_quotas_with_remove_header(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={ 'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_REMOVE_ACCOUNT_META_QUOTA_BYTES': 'True'}) res = req.get_response(app) self.assertEquals(res.status_int, 403) def test_delete_quotas_reseller(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_ACCOUNT_META_QUOTA_BYTES': '', 'reseller_request': True}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_delete_quotas_with_remove_header_reseller(self): headers = [('x-account-bytes-used', '0'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1/a', environ={ 'REQUEST_METHOD': 'POST', 'swift.cache': cache, 'HTTP_X_REMOVE_ACCOUNT_META_QUOTA_BYTES': 'True', 'reseller_request': True}) res = req.get_response(app) self.assertEquals(res.status_int, 200) def test_invalid_request_exception(self): headers = [('x-account-bytes-used', '1000'), ] app = account_quotas.AccountQuotaMiddleware(FakeApp(headers)) cache = FakeCache(None) req = Request.blank('/v1', environ={'REQUEST_METHOD': 'PUT', 'swift.cache': cache}) res = req.get_response(app) # Response code of 200 because authentication itself is not done here self.assertEquals(res.status_int, 200) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_container_sync.py0000664000175400017540000002143512323703611026064 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import shutil import tempfile import unittest import uuid from swift.common import swob from swift.common.middleware import container_sync from swift.proxy.controllers.base import _get_cache_key from swift.proxy.controllers.info import InfoController class FakeApp(object): def __call__(self, env, start_response): if env.get('PATH_INFO') == '/info': controller = InfoController( app=None, version=None, expose_info=True, disallowed_sections=[], admin_key=None) handler = getattr(controller, env.get('REQUEST_METHOD')) return handler(swob.Request(env))(env, start_response) if env.get('swift.authorize_override'): body = 'Response to Authorized Request' else: body = 'Pass-Through Response' start_response('200 OK', [('Content-Length', str(len(body)))]) return body class TestContainerSync(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() with open( os.path.join(self.tempdir, 'container-sync-realms.conf'), 'w') as fp: fp.write(''' [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b key2 = 1a0a5a0cbd66448084089304442d6776 cluster_dfw1 = http://dfw1.host/v1/ ''') self.app = FakeApp() self.conf = {'swift_dir': self.tempdir} self.sync = container_sync.ContainerSync(self.app, self.conf) def tearDown(self): shutil.rmtree(self.tempdir, ignore_errors=1) def test_pass_through(self): req = swob.Request.blank('/v1/a/c') resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.assertEqual(resp.body, 'Pass-Through Response') def test_not_enough_args(self): req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'a'}) resp = req.get_response(self.sync) self.assertEqual(resp.status, '401 Unauthorized') self.assertEqual( resp.body, 'X-Container-Sync-Auth header not valid; contact cluster operator ' 'for support.') self.assertTrue( 'cs:not-3-args' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_realm_miss(self): req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'invalid nonce sig'}) resp = req.get_response(self.sync) self.assertEqual(resp.status, '401 Unauthorized') self.assertEqual( resp.body, 'X-Container-Sync-Auth header not valid; contact cluster operator ' 'for support.') self.assertTrue( 'cs:no-local-realm-key' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_user_key_miss(self): req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'}) resp = req.get_response(self.sync) self.assertEqual(resp.status, '401 Unauthorized') self.assertEqual( resp.body, 'X-Container-Sync-Auth header not valid; contact cluster operator ' 'for support.') self.assertTrue( 'cs:no-local-user-key' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_invalid_sig(self): req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce sig'}) req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '401 Unauthorized') self.assertEqual( resp.body, 'X-Container-Sync-Auth header not valid; contact cluster operator ' 'for support.') self.assertTrue( 'cs:invalid-sig' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_valid_sig(self): sig = self.sync.realms_conf.get_sig( 'GET', '/v1/a/c', '0', 'nonce', self.sync.realms_conf.key('US'), 'abc') req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig}) req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.assertEqual(resp.body, 'Response to Authorized Request') self.assertTrue( 'cs:valid' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_valid_sig2(self): sig = self.sync.realms_conf.get_sig( 'GET', '/v1/a/c', '0', 'nonce', self.sync.realms_conf.key2('US'), 'abc') req = swob.Request.blank( '/v1/a/c', headers={'x-container-sync-auth': 'US nonce ' + sig}) req.environ[_get_cache_key('a', 'c')[1]] = {'sync_key': 'abc'} resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.assertEqual(resp.body, 'Response to Authorized Request') self.assertTrue( 'cs:valid' in req.environ.get('swift.log_info'), req.environ.get('swift.log_info')) def test_info(self): req = swob.Request.blank('/info') resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') result = json.loads(resp.body) self.assertEqual( result.get('container_sync'), {'realms': {'US': {'clusters': {'DFW1': {}}}}}) def test_info_always_fresh(self): req = swob.Request.blank('/info') resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') result = json.loads(resp.body) self.assertEqual( result.get('container_sync'), {'realms': {'US': {'clusters': {'DFW1': {}}}}}) with open( os.path.join(self.tempdir, 'container-sync-realms.conf'), 'w') as fp: fp.write(''' [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b key2 = 1a0a5a0cbd66448084089304442d6776 cluster_dfw1 = http://dfw1.host/v1/ [UK] key = 400b3b357a80413f9d956badff1d9dfe cluster_lon3 = http://lon3.host/v1/ ''') self.sync.realms_conf.reload() req = swob.Request.blank('/info') resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') result = json.loads(resp.body) self.assertEqual( result.get('container_sync'), {'realms': { 'US': {'clusters': {'DFW1': {}}}, 'UK': {'clusters': {'LON3': {}}}}}) def test_allow_full_urls_setting(self): req = swob.Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-container-sync-to': 'http://host/v1/a/c'}) resp = req.get_response(self.sync) self.assertEqual(resp.status, '200 OK') self.conf = {'swift_dir': self.tempdir, 'allow_full_urls': 'false'} self.sync = container_sync.ContainerSync(self.app, self.conf) req = swob.Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-container-sync-to': 'http://host/v1/a/c'}) resp = req.get_response(self.sync) self.assertEqual(resp.status, '400 Bad Request') self.assertEqual( resp.body, 'Full URLs are not allowed for X-Container-Sync-To values. Only ' 'realm values of the format //realm/cluster/account/container are ' 'allowed.\n') def test_filter(self): app = FakeApp() unique = uuid.uuid4().hex sync = container_sync.filter_factory( {'global': 'global_value', 'swift_dir': unique}, **{'local': 'local_value'})(app) self.assertEqual(sync.app, app) self.assertEqual(sync.conf, { 'global': 'global_value', 'swift_dir': unique, 'local': 'local_value'}) req = swob.Request.blank('/info') resp = req.get_response(sync) self.assertEqual(resp.status, '200 OK') result = json.loads(resp.body) self.assertEqual(result.get('container_sync'), {'realms': {}}) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/helpers.py0000664000175400017540000000750012323703611023446 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # This stuff can't live in test/unit/__init__.py due to its swob dependency. from copy import deepcopy from hashlib import md5 from swift.common import swob from swift.common.utils import split_path class FakeSwift(object): """ A good-enough fake Swift proxy server to use in testing middleware. """ def __init__(self): self._calls = [] self.req_method_paths = [] self.swift_sources = [] self.uploaded = {} # mapping of (method, path) --> (response class, headers, body) self._responses = {} def __call__(self, env, start_response): method = env['REQUEST_METHOD'] path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize']() if resp: return resp(env, start_response) headers = swob.Request(env).headers self._calls.append((method, path, headers)) self.swift_sources.append(env.get('swift.source')) try: resp_class, raw_headers, body = self._responses[(method, path)] headers = swob.HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._responses[ (method, env['PATH_INFO'])] headers = swob.HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, _ = self._responses[('GET', path)] body = None headers = swob.HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: print "Didn't find %r in allowed responses" % ((method, path),) raise # simulate object PUT if method == 'PUT' and obj: input = env['wsgi.input'].read() etag = md5(input).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(input)) # keep it for subsequent GET requests later self.uploaded[path] = (deepcopy(headers), input) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # range requests ought to work, hence conditional_response=True req = swob.Request(env) resp = resp_class(req=req, headers=headers, body=body, conditional_response=True) return resp(env, start_response) @property def calls(self): return [(method, path) for method, path, headers in self._calls] @property def calls_with_headers(self): return self._calls @property def call_count(self): return len(self._calls) def register(self, method, path, response_class, headers, body): self._responses[(method, path)] = (response_class, headers, body) swift-1.13.1/test/unit/common/middleware/test_staticweb.py0000664000175400017540000010333712323703611025035 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. try: import simplejson as json except ImportError: import json import json as stdlib_json import unittest import mock from swift.common.swob import Request, Response from swift.common.middleware import staticweb meta_map = { 'c1': {'status': 401}, 'c2': {}, 'c3': {'meta': {'web-index': 'index.html', 'web-listings': 't'}}, 'c3b': {'meta': {'web-index': 'index.html', 'web-listings': 't'}}, 'c4': {'meta': {'web-index': 'index.html', 'web-error': 'error.html', 'web-listings': 't', 'web-listings-css': 'listing.css', 'web-directory-type': 'text/dir'}}, 'c5': {'meta': {'web-index': 'index.html', 'web-error': 'error.html', 'web-listings': 't', 'web-listings-css': 'listing.css'}}, 'c6': {'meta': {'web-listings': 't'}}, 'c7': {'meta': {'web-listings': 'f'}}, 'c8': {'meta': {'web-error': 'error.html', 'web-listings': 't', 'web-listings-css': 'http://localhost/stylesheets/listing.css'}}, 'c9': {'meta': {'web-error': 'error.html', 'web-listings': 't', 'web-listings-css': '/absolute/listing.css'}}, 'c10': {'meta': {'web-listings': 't'}}, 'c11': {'meta': {'web-index': 'index.html'}}, 'c11a': {'meta': {'web-index': 'index.html', 'web-directory-type': 'text/directory'}}, 'c12': {'meta': {'web-index': 'index.html', 'web-error': 'error.html'}}, 'c13': {'meta': {'web-listings': 'f', 'web-listings-css': 'listing.css'}}, } def mock_get_container_info(env, app, swift_source='SW'): container = env['PATH_INFO'].rstrip('/').split('/')[3] container_info = meta_map[container] container_info.setdefault('status', 200) container_info.setdefault('read_acl', '.r:*') return container_info class FakeApp(object): def __init__(self, status_headers_body_iter=None): self.calls = 0 self.get_c4_called = False def __call__(self, env, start_response): self.calls += 1 if env['PATH_INFO'] == '/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1': return Response( status='412 Precondition Failed')(env, start_response) elif env['PATH_INFO'] == '/v1/a': return Response(status='401 Unauthorized')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c1': return Response(status='401 Unauthorized')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c2': return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c2/one.txt': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3': return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/index.html': return Response(status='200 Ok', body='''

Test main index.html file.

Visit subdir.

Don't visit subdir2 because it doesn't really exist.

Visit subdir3.

Visit subdir3/subsubdir.

''')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3b': return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3b/index.html': resp = Response(status='204 No Content') resp.app_iter = iter([]) return resp(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdir3/subsubdir/index.html': return Response(status='200 Ok', body='index file')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdirx/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdirx/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdiry/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdiry/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdirz': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/subdirz/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/unknown': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c3/unknown/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4': self.get_c4_called = True return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/one.txt': return Response( status='200 Ok', headers={'x-object-meta-test': 'value'}, body='1')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/two.txt': return Response(status='503 Service Unavailable')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/subdir/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/subdir/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/unknown': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/unknown/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c4/404error.html': return Response(status='200 Ok', body='''

Chrome's 404 fancy-page sucks.

'''.strip())(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5': return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5/index.html': return Response(status='503 Service Unavailable')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5/503error.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5/unknown': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5/unknown/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5/404error.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c6': return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c6/subdir': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c7', '/v1/a/c7/'): return self.listing(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c8', '/v1/a/c8/'): return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c8/subdir/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c9', '/v1/a/c9/'): return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c9/subdir/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c10', '/v1/a/c10/'): return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c10/\xe2\x98\x83/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c11', '/v1/a/c11/'): return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11/subdir/': return Response(status='200 Ok', headers={ 'Content-Type': 'application/directory'})( env, start_response) elif env['PATH_INFO'] == '/v1/a/c11/subdir/index.html': return Response(status='200 Ok', body='''

c11 subdir index

'''.strip())(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11/subdir2/': return Response(status='200 Ok', headers={'Content-Type': 'application/directory'})(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11/subdir2/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] in ('/v1/a/c11a', '/v1/a/c11a/'): return self.listing(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir/': return Response(status='200 Ok', headers={'Content-Type': 'text/directory'})(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir2/': return Response(status='200 Ok', headers={'Content-Type': 'application/directory'})(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir2/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir3/': return Response(status='200 Ok', headers={'Content-Type': 'not_a/directory'})(env, start_response) elif env['PATH_INFO'] == '/v1/a/c11a/subdir3/index.html': return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c12/index.html': return Response(status='200 Ok', body='index file')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c12/200error.html': return Response(status='200 Ok', body='error file')(env, start_response) else: raise Exception('Unknown path %r' % env['PATH_INFO']) def listing(self, env, start_response): headers = {'x-container-read': '.r:*'} if ((env['PATH_INFO'] in ( '/v1/a/c3', '/v1/a/c4', '/v1/a/c8', '/v1/a/c9')) and (env['QUERY_STRING'] == 'delimiter=/&format=json&prefix=subdir/')): headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'X-Container-Read': '.r:*', 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.709100"}, {"name":"subdir/2.txt", "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.734140"}, {"subdir":"subdir3/subsubdir/"}] '''.strip() elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ 'delimiter=/&format=json&prefix=subdiry/': headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'X-Container-Read': '.r:*', 'Content-Type': 'application/json; charset=utf-8'}) body = '[]' elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ 'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/': headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'X-Container-Read': '.r:*', 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdirz/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.709100"}] '''.strip() elif env['PATH_INFO'] == '/v1/a/c6' and env['QUERY_STRING'] == \ 'limit=1&format=json&delimiter=/&limit=1&prefix=subdir/': headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'X-Container-Read': '.r:*', 'X-Container-Web-Listings': 't', 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.709100"}] '''.strip() elif env['PATH_INFO'] == '/v1/a/c10' and ( env['QUERY_STRING'] == 'delimiter=/&format=json&prefix=%E2%98%83/' or env['QUERY_STRING'] == 'delimiter=/&format=json&prefix=%E2%98%83/%E2%98%83/'): headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'X-Container-Read': '.r:*', 'X-Container-Web-Listings': 't', 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"\u2603/\u2603/one.txt", "hash":"73f1dd69bacbf0847cc9cffa3c6b23a1", "bytes":22, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.709100"}, {"subdir":"\u2603/\u2603/"}] '''.strip() elif 'prefix=' in env['QUERY_STRING']: return Response(status='204 No Content')(env, start_response) elif 'format=json' in env['QUERY_STRING']: headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"401error.html", "hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110, "content_type":"text/html", "last_modified":"2011-03-24T04:27:52.713710"}, {"name":"404error.html", "hash":"62dcec9c34ed2b347d94e6ca707aff8c", "bytes":130, "content_type":"text/html", "last_modified":"2011-03-24T04:27:52.720850"}, {"name":"index.html", "hash":"8b469f2ca117668a5131fe9ee0815421", "bytes":347, "content_type":"text/html", "last_modified":"2011-03-24T04:27:52.683590"}, {"name":"listing.css", "hash":"7eab5d169f3fcd06a08c130fa10c5236", "bytes":17, "content_type":"text/css", "last_modified":"2011-03-24T04:27:52.721610"}, {"name":"one.txt", "hash":"73f1dd69bacbf0847cc9cffa3c6b23a1", "bytes":22, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.722270"}, {"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.709100"}, {"name":"subdir/2.txt", "hash":"c85c1dcd19cf5cbac84e6043c31bb63e", "bytes":20, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.734140"}, {"name":"subdir/\u2603.txt", "hash":"7337d028c093130898d937c319cc9865", "bytes":72981, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.735460"}, {"name":"subdir2", "hash":"d41d8cd98f00b204e9800998ecf8427e", "bytes":0, "content_type":"text/directory", "last_modified":"2011-03-24T04:27:52.676690"}, {"name":"subdir3/subsubdir/index.html", "hash":"04eea67110f883b1a5c97eb44ccad08c", "bytes":72, "content_type":"text/html", "last_modified":"2011-03-24T04:27:52.751260"}, {"name":"two.txt", "hash":"10abb84c63a5cff379fdfd6385918833", "bytes":22, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.825110"}, {"name":"\u2603/\u2603/one.txt", "hash":"73f1dd69bacbf0847cc9cffa3c6b23a1", "bytes":22, "content_type":"text/plain", "last_modified":"2011-03-24T04:27:52.935560"}] '''.strip() else: headers.update({'X-Container-Object-Count': '12', 'X-Container-Bytes-Used': '73763', 'Content-Type': 'text/plain; charset=utf-8'}) body = '\n'.join(['401error.html', '404error.html', 'index.html', 'listing.css', 'one.txt', 'subdir/1.txt', 'subdir/2.txt', u'subdir/\u2603.txt', 'subdir2', 'subdir3/subsubdir/index.html', 'two.txt', u'\u2603/\u2603/one.txt']) return Response(status='200 Ok', headers=headers, body=body)(env, start_response) class TestStaticWeb(unittest.TestCase): def setUp(self): self.app = FakeApp() self.test_staticweb = staticweb.filter_factory({})(self.app) self._orig_get_container_info = staticweb.get_container_info staticweb.get_container_info = mock_get_container_info def tearDown(self): staticweb.get_container_info = self._orig_get_container_info def test_app_set(self): app = FakeApp() sw = staticweb.filter_factory({})(app) self.assertEquals(sw.app, app) def test_conf_set(self): conf = {'blah': 1} sw = staticweb.filter_factory(conf)(FakeApp()) self.assertEquals(sw.conf, conf) def test_root(self): resp = Request.blank('/').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 404) def test_version(self): resp = Request.blank('/v1').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 412) def test_account(self): resp = Request.blank('/v1/a').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 401) def test_container1(self): resp = Request.blank('/v1/a/c1').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 401) def test_container1_web_mode_explicitly_off(self): resp = Request.blank('/v1/a/c1', headers={'x-web-mode': 'false'}).get_response( self.test_staticweb) self.assertEquals(resp.status_int, 401) def test_container1_web_mode_explicitly_on(self): resp = Request.blank('/v1/a/c1', headers={'x-web-mode': 'true'}).get_response( self.test_staticweb) self.assertEquals(resp.status_int, 404) def test_container2(self): resp = Request.blank('/v1/a/c2').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(len(resp.body.split('\n')), int(resp.headers['x-container-object-count'])) def test_container2_web_mode_explicitly_off(self): resp = Request.blank( '/v1/a/c2', headers={'x-web-mode': 'false'}).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(len(resp.body.split('\n')), int(resp.headers['x-container-object-count'])) def test_container2_web_mode_explicitly_on(self): resp = Request.blank( '/v1/a/c2', headers={'x-web-mode': 'true'}).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 404) def test_container2onetxt(self): resp = Request.blank( '/v1/a/c2/one.txt').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 404) def test_container2json(self): resp = Request.blank( '/v1/a/c2?format=json').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(len(json.loads(resp.body)), int(resp.headers['x-container-object-count'])) def test_container2json_web_mode_explicitly_off(self): resp = Request.blank( '/v1/a/c2?format=json', headers={'x-web-mode': 'false'}).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(len(json.loads(resp.body)), int(resp.headers['x-container-object-count'])) def test_container2json_web_mode_explicitly_on(self): resp = Request.blank( '/v1/a/c2?format=json', headers={'x-web-mode': 'true'}).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 404) def test_container3(self): resp = Request.blank('/v1/a/c3').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 301) self.assertEquals(resp.headers['location'], 'http://localhost/v1/a/c3/') def test_container3indexhtml(self): resp = Request.blank('/v1/a/c3/').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assert_('Test main index.html file.' in resp.body) def test_container3subsubdir(self): resp = Request.blank( '/v1/a/c3/subdir3/subsubdir').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 301) def test_container3subsubdircontents(self): resp = Request.blank( '/v1/a/c3/subdir3/subsubdir/').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body, 'index file') def test_container3subdir(self): resp = Request.blank( '/v1/a/c3/subdir/').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assert_('Listing of /v1/a/c3/subdir/' in resp.body) self.assert_('' in resp.body) self.assert_('' not in resp.body) self.assert_('c11 subdir index' in resp.body) def test_container11subdirmarkermatchdirtype(self): resp = Request.blank('/v1/a/c11a/subdir/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 404) self.assert_('Index File Not Found' in resp.body) def test_container11subdirmarkeraltdirtype(self): resp = Request.blank('/v1/a/c11a/subdir2/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 200) def test_container11subdirmarkerinvaliddirtype(self): resp = Request.blank('/v1/a/c11a/subdir3/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 200) def test_container12unredirectedrequest(self): resp = Request.blank('/v1/a/c12/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assert_('index file' in resp.body) def test_container_404_has_css(self): resp = Request.blank('/v1/a/c13/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 404) self.assert_('listing.css' in resp.body) def test_container_404_has_no_css(self): resp = Request.blank('/v1/a/c7/').get_response( self.test_staticweb) self.assertEquals(resp.status_int, 404) self.assert_('listing.css' not in resp.body) self.assert_(' /cont/object etagoftheobjectsegment 100 ''' test_json_data = json.dumps([{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100}]) def fake_start_response(*args, **kwargs): pass def md5hex(s): return hashlib.md5(s).hexdigest() class SloTestCase(unittest.TestCase): def setUp(self): self.app = FakeSwift() self.slo = slo.filter_factory({})(self.app) self.slo.min_segment_size = 1 def call_app(self, req, app=None, expect_exception=False): if app is None: app = self.app req.headers.setdefault("User-Agent", "Mozzarella Foxfire") status = [None] headers = [None] def start_response(s, h, ei=None): status[0] = s headers[0] = h body_iter = app(req.environ, start_response) body = '' caught_exc = None try: for chunk in body_iter: body += chunk except Exception as exc: if expect_exception: caught_exc = exc else: raise if expect_exception: return status[0], headers[0], body, caught_exc else: return status[0], headers[0], body def call_slo(self, req, **kwargs): return self.call_app(req, app=self.slo, **kwargs) class TestSloMiddleware(SloTestCase): def setUp(self): super(TestSloMiddleware, self).setUp() self.app.register( 'GET', '/', swob.HTTPOk, {}, 'passed') self.app.register( 'PUT', '/', swob.HTTPOk, {}, 'passed') def test_handle_multipart_no_obj(self): req = Request.blank('/') resp_iter = self.slo(req.environ, fake_start_response) self.assertEquals(self.app.calls, [('GET', '/')]) self.assertEquals(''.join(resp_iter), 'passed') def test_slo_header_assigned(self): req = Request.blank( '/v1/a/c/o', headers={'x-static-large-object': "true"}, environ={'REQUEST_METHOD': 'PUT'}) resp = ''.join(self.slo(req.environ, fake_start_response)) self.assert_( resp.startswith('X-Static-Large-Object is a reserved header')) def test_parse_input(self): self.assertRaises(HTTPException, slo.parse_input, 'some non json') data = json.dumps( [{'path': '/cont/object', 'etag': 'etagoftheobjecitsegment', 'size_bytes': 100}]) self.assertEquals('/cont/object', slo.parse_input(data)[0]['path']) class TestSloPutManifest(SloTestCase): def setUp(self): super(TestSloPutManifest, self).setUp() self.app.register( 'GET', '/', swob.HTTPOk, {}, 'passed') self.app.register( 'PUT', '/', swob.HTTPOk, {}, 'passed') self.app.register( 'HEAD', '/v1/AUTH_test/cont/object', swob.HTTPOk, {'Content-Length': '100', 'Etag': 'etagoftheobjectsegment'}, None) self.app.register( 'HEAD', '/v1/AUTH_test/cont/object\xe2\x99\xa1', swob.HTTPOk, {'Content-Length': '100', 'Etag': 'etagoftheobjectsegment'}, None) self.app.register( 'HEAD', '/v1/AUTH_test/cont/small_object', swob.HTTPOk, {'Content-Length': '10', 'Etag': 'etagoftheobjectsegment'}, None) self.app.register( 'PUT', '/v1/AUTH_test/c/man', swob.HTTPCreated, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/c/man', swob.HTTPNoContent, {}, None) self.app.register( 'HEAD', '/v1/AUTH_test/checktest/a_1', swob.HTTPOk, {'Content-Length': '1', 'Etag': 'a'}, None) self.app.register( 'HEAD', '/v1/AUTH_test/checktest/badreq', swob.HTTPBadRequest, {}, None) self.app.register( 'HEAD', '/v1/AUTH_test/checktest/b_2', swob.HTTPOk, {'Content-Length': '2', 'Etag': 'b', 'Last-Modified': 'Fri, 01 Feb 2012 20:38:36 GMT'}, None) self.app.register( 'GET', '/v1/AUTH_test/checktest/slob', swob.HTTPOk, {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag'}, None) self.app.register( 'PUT', '/v1/AUTH_test/checktest/man_3', swob.HTTPCreated, {}, None) def test_put_manifest_too_quick_fail(self): req = Request.blank('/v1/a/c/o') req.content_length = self.slo.max_manifest_size + 1 try: self.slo.handle_multipart_put(req, fake_start_response) except HTTPException as e: pass self.assertEquals(e.status_int, 413) with patch.object(self.slo, 'max_manifest_segments', 0): req = Request.blank('/v1/a/c/o', body=test_json_data) e = None try: self.slo.handle_multipart_put(req, fake_start_response) except HTTPException as e: pass self.assertEquals(e.status_int, 413) with patch.object(self.slo, 'min_segment_size', 1000): req = Request.blank('/v1/a/c/o', body=test_json_data) try: self.slo.handle_multipart_put(req, fake_start_response) except HTTPException as e: pass self.assertEquals(e.status_int, 400) req = Request.blank('/v1/a/c/o', headers={'X-Copy-From': 'lala'}) try: self.slo.handle_multipart_put(req, fake_start_response) except HTTPException as e: pass self.assertEquals(e.status_int, 405) # ignores requests to / req = Request.blank( '/?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=test_json_data) self.assertEquals( self.slo.handle_multipart_put(req, fake_start_response), ['passed']) def test_handle_multipart_put_success(self): req = Request.blank( '/v1/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, body=test_json_data) self.assertTrue('X-Static-Large-Object' not in req.headers) def my_fake_start_response(*args, **kwargs): gen_etag = '"' + md5('etagoftheobjectsegment').hexdigest() + '"' self.assertTrue(('Etag', gen_etag) in args[1]) self.slo(req.environ, my_fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) def test_handle_multipart_put_success_allow_small_last_segment(self): with patch.object(self.slo, 'min_segment_size', 50): test_json_data = json.dumps([{'path': '/cont/object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100}, {'path': '/cont/small_object', 'etag': 'etagoftheobjectsegment', 'size_bytes': 10}]) req = Request.blank( '/v1/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, body=test_json_data) self.assertTrue('X-Static-Large-Object' not in req.headers) self.slo(req.environ, fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) def test_handle_multipart_put_success_unicode(self): test_json_data = json.dumps([{'path': u'/cont/object\u2661', 'etag': 'etagoftheobjectsegment', 'size_bytes': 100}]) req = Request.blank( '/v1/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, body=test_json_data) self.assertTrue('X-Static-Large-Object' not in req.headers) self.slo(req.environ, fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) self.assertTrue(req.environ['PATH_INFO'], '/cont/object\xe2\x99\xa1') def test_handle_multipart_put_no_xml(self): req = Request.blank( '/test_good/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, body=test_xml_data) no_xml = self.slo(req.environ, fake_start_response) self.assertEquals(no_xml, ['Manifest must be valid json.']) def test_handle_multipart_put_bad_data(self): bad_data = json.dumps([{'path': '/cont/object', 'etag': 'etagoftheobj', 'size_bytes': 'lala'}]) req = Request.blank( '/test_good/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=bad_data) self.assertRaises(HTTPException, self.slo.handle_multipart_put, req, fake_start_response) for bad_data in [ json.dumps([{'path': '/cont', 'etag': 'etagoftheobj', 'size_bytes': 100}]), json.dumps('asdf'), json.dumps(None), json.dumps(5), 'not json', '1234', None, '', json.dumps({'path': None}), json.dumps([{'path': '/cont/object', 'etag': None, 'size_bytes': 12}]), json.dumps([{'path': '/cont/object', 'etag': 'asdf', 'size_bytes': 'sd'}]), json.dumps([{'path': 12, 'etag': 'etagoftheobj', 'size_bytes': 100}]), json.dumps([{'path': u'/cont/object\u2661', 'etag': 'etagoftheobj', 'size_bytes': 100}]), json.dumps([{'path': 12, 'size_bytes': 100}]), json.dumps([{'path': 12, 'size_bytes': 100}]), json.dumps([{'path': None, 'etag': 'etagoftheobj', 'size_bytes': 100}])]: req = Request.blank( '/v1/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=bad_data) self.assertRaises(HTTPException, self.slo.handle_multipart_put, req, fake_start_response) def test_handle_multipart_put_check_data(self): good_data = json.dumps( [{'path': '/checktest/a_1', 'etag': 'a', 'size_bytes': '1'}, {'path': '/checktest/b_2', 'etag': 'b', 'size_bytes': '2'}]) req = Request.blank( '/v1/AUTH_test/checktest/man_3?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=good_data) status, headers, body = self.call_slo(req) self.assertEquals(self.app.call_count, 3) # go behind SLO's back and see what actually got stored req = Request.blank( # this string looks weird, but it's just an artifact # of FakeSwift '/v1/AUTH_test/checktest/man_3?multipart-manifest=put', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_app(req) headers = dict(headers) manifest_data = json.loads(body) self.assert_(headers['Content-Type'].endswith(';swift_bytes=3')) self.assertEquals(len(manifest_data), 2) self.assertEquals(manifest_data[0]['hash'], 'a') self.assertEquals(manifest_data[0]['bytes'], 1) self.assert_(not manifest_data[0]['last_modified'].startswith('2012')) self.assert_(manifest_data[1]['last_modified'].startswith('2012')) def test_handle_multipart_put_check_data_bad(self): bad_data = json.dumps( [{'path': '/checktest/a_1', 'etag': 'a', 'size_bytes': '2'}, {'path': '/checktest/badreq', 'etag': 'a', 'size_bytes': '1'}, {'path': '/checktest/b_2', 'etag': 'not-b', 'size_bytes': '2'}, {'path': '/checktest/slob', 'etag': 'not-slob', 'size_bytes': '2'}]) req = Request.blank( '/v1/AUTH_test/checktest/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'application/json'}, body=bad_data) status, headers, body = self.call_slo(req) self.assertEquals(self.app.call_count, 5) errors = json.loads(body)['Errors'] self.assertEquals(len(errors), 5) self.assertEquals(errors[0][0], '/checktest/a_1') self.assertEquals(errors[0][1], 'Size Mismatch') self.assertEquals(errors[1][0], '/checktest/badreq') self.assertEquals(errors[1][1], '400 Bad Request') self.assertEquals(errors[2][0], '/checktest/b_2') self.assertEquals(errors[2][1], 'Etag Mismatch') self.assertEquals(errors[3][0], '/checktest/slob') self.assertEquals(errors[3][1], 'Size Mismatch') self.assertEquals(errors[4][0], '/checktest/slob') self.assertEquals(errors[4][1], 'Etag Mismatch') class TestSloDeleteManifest(SloTestCase): def setUp(self): super(TestSloDeleteManifest, self).setUp() _submanifest_data = json.dumps( [{'name': '/deltest/b_2', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest/c_3', 'hash': 'b', 'bytes': '2'}]) self.app.register( 'GET', '/v1/AUTH_test/deltest/man_404', swob.HTTPNotFound, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/man', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/deltest/gone', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest/b_2', 'hash': 'b', 'bytes': '2'}])) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/man', swob.HTTPNoContent, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/man-all-there', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/deltest/b_2', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest/c_3', 'hash': 'b', 'bytes': '2'}])) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/man-all-there', swob.HTTPNoContent, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/gone', swob.HTTPNotFound, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/a_1', swob.HTTPOk, {'Content-Length': '1'}, 'a') self.app.register( 'DELETE', '/v1/AUTH_test/deltest/a_1', swob.HTTPNoContent, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/b_2', swob.HTTPNoContent, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/c_3', swob.HTTPNoContent, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/d_3', swob.HTTPNoContent, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/manifest-with-submanifest', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/deltest/a_1', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest/submanifest', 'sub_slo': True, 'hash': 'submanifest-etag', 'bytes': len(_submanifest_data)}, {'name': '/deltest/d_3', 'hash': 'd', 'bytes': '3'}])) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/manifest-with-submanifest', swob.HTTPNoContent, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/submanifest', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, _submanifest_data) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/submanifest', swob.HTTPNoContent, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/manifest-missing-submanifest', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/deltest/a_1', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest/missing-submanifest', 'hash': 'a', 'bytes': '2', 'sub_slo': True}, {'name': '/deltest/d_3', 'hash': 'd', 'bytes': '3'}])) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/manifest-missing-submanifest', swob.HTTPNoContent, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/missing-submanifest', swob.HTTPNotFound, {}, None) self.app.register( 'GET', '/v1/AUTH_test/deltest/manifest-badjson', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, "[not {json (at ++++all") self.app.register( 'GET', '/v1/AUTH_test/deltest/manifest-with-unauth-segment', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/deltest/a_1', 'hash': 'a', 'bytes': '1'}, {'name': '/deltest-unauth/q_17', 'hash': '11', 'bytes': '17'}])) self.app.register( 'DELETE', '/v1/AUTH_test/deltest/manifest-with-unauth-segment', swob.HTTPNoContent, {}, None) self.app.register( 'DELETE', '/v1/AUTH_test/deltest-unauth/q_17', swob.HTTPUnauthorized, {}, None) def test_handle_multipart_delete_man(self): req = Request.blank( '/v1/AUTH_test/deltest/man', environ={'REQUEST_METHOD': 'DELETE'}) self.slo(req.environ, fake_start_response) self.assertEquals(self.app.call_count, 1) def test_handle_multipart_delete_bad_utf8(self): req = Request.blank( '/v1/AUTH_test/deltest/man\xff\xfe?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) self.assertEquals(status, '200 OK') resp_data = json.loads(body) self.assertEquals(resp_data['Response Status'], '412 Precondition Failed') def test_handle_multipart_delete_whole_404(self): req = Request.blank( '/v1/AUTH_test/deltest/man_404?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/man_404?multipart-manifest=get')]) self.assertEquals(resp_data['Response Status'], '200 OK') self.assertEquals(resp_data['Response Body'], '') self.assertEquals(resp_data['Number Deleted'], 0) self.assertEquals(resp_data['Number Not Found'], 1) self.assertEquals(resp_data['Errors'], []) def test_handle_multipart_delete_segment_404(self): req = Request.blank( '/v1/AUTH_test/deltest/man?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/man?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/gone?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/b_2?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/man?multipart-manifest=delete')]) self.assertEquals(resp_data['Response Status'], '200 OK') self.assertEquals(resp_data['Number Deleted'], 2) self.assertEquals(resp_data['Number Not Found'], 1) def test_handle_multipart_delete_whole(self): req = Request.blank( '/v1/AUTH_test/deltest/man-all-there?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE'}) self.call_slo(req) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/man-all-there?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/b_2?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/c_3?multipart-manifest=delete'), ('DELETE', ('/v1/AUTH_test/deltest/' + 'man-all-there?multipart-manifest=delete'))]) def test_handle_multipart_delete_nested(self): req = Request.blank( '/v1/AUTH_test/deltest/manifest-with-submanifest?' + 'multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE'}) self.call_slo(req) self.assertEquals( set(self.app.calls), set([('GET', '/v1/AUTH_test/deltest/' + 'manifest-with-submanifest?multipart-manifest=get'), ('GET', '/v1/AUTH_test/deltest/' + 'submanifest?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/a_1?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/b_2?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/c_3?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/' + 'submanifest?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/d_3?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/' + 'manifest-with-submanifest?multipart-manifest=delete')])) def test_handle_multipart_delete_nested_too_many_segments(self): req = Request.blank( '/v1/AUTH_test/deltest/manifest-with-submanifest?' + 'multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) with patch.object(slo, 'MAX_BUFFERED_SLO_SEGMENTS', 1): status, headers, body = self.call_slo(req) self.assertEquals(status, '200 OK') resp_data = json.loads(body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Response Body'], 'Too many buffered slo segments to delete.') def test_handle_multipart_delete_nested_404(self): req = Request.blank( '/v1/AUTH_test/deltest/manifest-missing-submanifest' + '?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/' + 'manifest-missing-submanifest?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/a_1?multipart-manifest=delete'), ('GET', '/v1/AUTH_test/deltest/' + 'missing-submanifest?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/d_3?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/' + 'manifest-missing-submanifest?multipart-manifest=delete')]) self.assertEquals(resp_data['Response Status'], '200 OK') self.assertEquals(resp_data['Response Body'], '') self.assertEquals(resp_data['Number Deleted'], 3) self.assertEquals(resp_data['Number Not Found'], 1) self.assertEquals(resp_data['Errors'], []) def test_handle_multipart_delete_nested_401(self): self.app.register( 'GET', '/v1/AUTH_test/deltest/submanifest', swob.HTTPUnauthorized, {}, None) req = Request.blank( ('/v1/AUTH_test/deltest/manifest-with-submanifest' + '?multipart-manifest=delete'), environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) self.assertEquals(status, '200 OK') resp_data = json.loads(body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Errors'], [['/deltest/submanifest', '401 Unauthorized']]) def test_handle_multipart_delete_nested_500(self): self.app.register( 'GET', '/v1/AUTH_test/deltest/submanifest', swob.HTTPServerError, {}, None) req = Request.blank( ('/v1/AUTH_test/deltest/manifest-with-submanifest' + '?multipart-manifest=delete'), environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) self.assertEquals(status, '200 OK') resp_data = json.loads(body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Errors'], [['/deltest/submanifest', 'Unable to load SLO manifest or segment.']]) def test_handle_multipart_delete_not_a_manifest(self): req = Request.blank( '/v1/AUTH_test/deltest/a_1?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/a_1?multipart-manifest=get')]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Response Body'], '') self.assertEquals(resp_data['Number Deleted'], 0) self.assertEquals(resp_data['Number Not Found'], 0) self.assertEquals(resp_data['Errors'], [['/deltest/a_1', 'Not an SLO manifest']]) def test_handle_multipart_delete_bad_json(self): req = Request.blank( '/v1/AUTH_test/deltest/manifest-badjson?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals(self.app.calls, [('GET', '/v1/AUTH_test/deltest/' + 'manifest-badjson?multipart-manifest=get')]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Response Body'], '') self.assertEquals(resp_data['Number Deleted'], 0) self.assertEquals(resp_data['Number Not Found'], 0) self.assertEquals(resp_data['Errors'], [['/deltest/manifest-badjson', 'Unable to load SLO manifest']]) def test_handle_multipart_delete_401(self): req = Request.blank( '/v1/AUTH_test/deltest/manifest-with-unauth-segment' + '?multipart-manifest=delete', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) resp_data = json.loads(body) self.assertEquals( self.app.calls, [('GET', '/v1/AUTH_test/deltest/' + 'manifest-with-unauth-segment?multipart-manifest=get'), ('DELETE', '/v1/AUTH_test/deltest/a_1?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest-unauth/' + 'q_17?multipart-manifest=delete'), ('DELETE', '/v1/AUTH_test/deltest/' + 'manifest-with-unauth-segment?multipart-manifest=delete')]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Response Body'], '') self.assertEquals(resp_data['Number Deleted'], 2) self.assertEquals(resp_data['Number Not Found'], 0) self.assertEquals(resp_data['Errors'], [['/deltest-unauth/q_17', '401 Unauthorized']]) class TestSloHeadManifest(SloTestCase): def setUp(self): super(TestSloHeadManifest, self).setUp() self._manifest_json = json.dumps([ {'name': '/gettest/seg01', 'bytes': '100', 'hash': 'seg01-hash', 'content_type': 'text/plain', 'last_modified': '2013-11-19T11:33:45.137446'}, {'name': '/gettest/seg02', 'bytes': '200', 'hash': 'seg02-hash', 'content_type': 'text/plain', 'last_modified': '2013-11-19T11:33:45.137447'}]) self.app.register( 'GET', '/v1/AUTH_test/headtest/man', swob.HTTPOk, {'Content-Length': str(len(self._manifest_json)), 'X-Static-Large-Object': 'true', 'Etag': md5(self._manifest_json).hexdigest()}, self._manifest_json) def test_etag_is_hash_of_segment_etags(self): req = Request.blank( '/v1/AUTH_test/headtest/man', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers.get('Etag', '').strip("'\""), md5("seg01-hashseg02-hash").hexdigest()) self.assertEqual(body, '') # it's a HEAD request, after all def test_etag_matching(self): etag = md5("seg01-hashseg02-hash").hexdigest() req = Request.blank( '/v1/AUTH_test/headtest/man', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': etag}) status, headers, body = self.call_slo(req) self.assertEqual(status, '304 Not Modified') class TestSloGetManifest(SloTestCase): def setUp(self): super(TestSloGetManifest, self).setUp() _bc_manifest_json = json.dumps( [{'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'bytes': '10', 'content_type': 'text/plain'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'bytes': '15', 'content_type': 'text/plain'}]) # some plain old objects self.app.register( 'GET', '/v1/AUTH_test/gettest/a_5', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex('a' * 5)}, 'a' * 5) self.app.register( 'GET', '/v1/AUTH_test/gettest/b_10', swob.HTTPOk, {'Content-Length': '10', 'Etag': md5hex('b' * 10)}, 'b' * 10) self.app.register( 'GET', '/v1/AUTH_test/gettest/c_15', swob.HTTPOk, {'Content-Length': '15', 'Etag': md5hex('c' * 15)}, 'c' * 15) self.app.register( 'GET', '/v1/AUTH_test/gettest/d_20', swob.HTTPOk, {'Content-Length': '20', 'Etag': md5hex('d' * 20)}, 'd' * 20) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc', swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus', 'Etag': md5hex(_bc_manifest_json)}, _bc_manifest_json) _abcd_manifest_json = json.dumps( [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/manifest-bc', 'sub_slo': True, 'content_type': 'application/json;swift_bytes=25', 'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)), 'bytes': len(_bc_manifest_json)}, {'name': '/gettest/d_20', 'hash': md5hex("d" * 20), 'content_type': 'text/plain', 'bytes': '20'}]) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-abcd', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'Etag': md5(_abcd_manifest_json).hexdigest()}, _abcd_manifest_json) self.manifest_abcd_etag = md5hex( md5hex("a" * 5) + md5hex(md5hex("b" * 10) + md5hex("c" * 15)) + md5hex("d" * 20)) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-badjson', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Fish': 'Bass'}, "[not {json (at ++++all") def test_get_manifest_passthrough(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc?multipart-manifest=get', environ={'REQUEST_METHOD': 'GET', 'HTTP_ACCEPT': 'application/json'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertTrue( ('Content-Type', 'application/json; charset=utf-8') in headers, headers) try: resp_data = json.loads(body) except json.JSONDecodeError: resp_data = None self.assertEqual( resp_data, [{'hash': md5hex('b' * 10), 'bytes': '10', 'name': '/gettest/b_10', 'content_type': 'text/plain'}, {'hash': md5hex('c' * 15), 'bytes': '15', 'name': '/gettest/c_15', 'content_type': 'text/plain'}], body) def test_get_nonmanifest_passthrough(self): req = Request.blank( '/v1/AUTH_test/gettest/a_5', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '200 OK') self.assertEqual(body, 'aaaaa') def test_get_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-bc', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) manifest_etag = md5hex(md5hex("b" * 10) + md5hex("c" * 15)) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') self.assertEqual(headers['Etag'], '"%s"' % manifest_etag) self.assertEqual(headers['X-Object-Meta-Plant'], 'Ficus') self.assertEqual(body, 'bbbbbbbbbbccccccccccccccc') for _, _, hdrs in self.app.calls_with_headers[1:]: ua = hdrs.get("User-Agent", "") self.assertTrue("SLO MultipartGET" in ua) self.assertFalse("SLO MultipartGET SLO MultipartGET" in ua) # the first request goes through unaltered first_ua = self.app.calls_with_headers[0][2].get("User-Agent") self.assertFalse( "SLO MultipartGET" in first_ua) def test_if_none_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': self.manifest_abcd_etag}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '304 Not Modified') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(body, '') def test_if_none_match_does_not_match(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': "not-%s" % self.manifest_abcd_etag}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') def test_if_match_matches(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': self.manifest_abcd_etag}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') def test_if_match_does_not_match(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': "not-%s" % self.manifest_abcd_etag}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(body, '') def test_if_match_matches_and_range(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': self.manifest_abcd_etag, 'Range': 'bytes=3-6'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '4') self.assertEqual(body, 'aabb') def test_get_manifest_with_submanifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '50') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag) self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') def test_range_get_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=3-17'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '15') self.assertTrue('Etag' not in headers) self.assertEqual(body, 'aabbbbbbbbbbccc') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) headers = [c[2] for c in self.app.calls_with_headers] self.assertEqual(headers[0].get('Range'), 'bytes=3-17') self.assertEqual(headers[1].get('Range'), None) self.assertEqual(headers[2].get('Range'), 'bytes=3-') self.assertEqual(headers[3].get('Range'), None) self.assertEqual(headers[4].get('Range'), None) self.assertEqual(headers[5].get('Range'), 'bytes=0-2') # we set swift.source for everything but the first request self.assertEqual(self.app.swift_sources, [None, 'SLO', 'SLO', 'SLO', 'SLO', 'SLO']) def test_range_get_includes_whole_manifest(self): # If the first range GET results in retrieval of the entire manifest # body (which we can detect by looking at Content-Range), then we # should not go make a second, non-ranged request just to retrieve the # same bytes again. req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-999999999'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')]) def test_range_get_beyond_manifest(self): big = 'e' * 1024 * 1024 big_etag = md5hex(big) self.app.register( 'GET', '/v1/AUTH_test/gettest/big_seg', swob.HTTPOk, {'Content-Type': 'application/foo', 'Etag': big_etag}, big) big_manifest = json.dumps( [{'name': '/gettest/big_seg', 'hash': big_etag, 'bytes': 1024 * 1024, 'content_type': 'application/foo'}]) self.app.register( 'GET', '/v1/AUTH_test/gettest/big_manifest', swob.HTTPOk, {'Content-Type': 'application/octet-stream', 'X-Static-Large-Object': 'true', 'Etag': md5(big_manifest).hexdigest()}, big_manifest) req = Request.blank( '/v1/AUTH_test/gettest/big_manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=100000-199999'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(body, 'e' * 100000) self.assertEqual( self.app.calls, [ # has Range header, gets 416 ('GET', '/v1/AUTH_test/gettest/big_manifest'), # retry the first one ('GET', '/v1/AUTH_test/gettest/big_manifest'), ('GET', '/v1/AUTH_test/gettest/big_seg?multipart-manifest=get')]) def test_range_get_bogus_content_range(self): # Just a little paranoia; Swift currently sends back valid # Content-Range headers, but if somehow someone sneaks an invalid one # in there, we'll ignore it. def content_range_breaker_factory(app): def content_range_breaker(env, start_response): req = swob.Request(env) resp = req.get_response(app) resp.headers['Content-Range'] = 'triscuits' return resp(env, start_response) return content_range_breaker self.slo = slo.filter_factory({})( content_range_breaker_factory(self.app)) req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-999999999'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')]) def test_range_get_manifest_on_segment_boundaries(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=5-29'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '25') self.assertTrue('Etag' not in headers) self.assertEqual(body, 'bbbbbbbbbbccccccccccccccc') self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) headers = [c[2] for c in self.app.calls_with_headers] self.assertEqual(headers[0].get('Range'), 'bytes=5-29') self.assertEqual(headers[1].get('Range'), None) self.assertEqual(headers[2].get('Range'), None) self.assertEqual(headers[3].get('Range'), None) self.assertEqual(headers[4].get('Range'), None) def test_range_get_manifest_first_byte(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-0'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '1') self.assertEqual(body, 'a') # Make sure we don't get any objects we don't need, including # submanifests. self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get')]) def test_range_get_manifest_sub_slo(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=25-30'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '6') self.assertEqual(body, 'cccccd') # Make sure we don't get any objects we don't need, including # submanifests. self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/d_20?multipart-manifest=get')]) def test_range_get_manifest_overlapping_end(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=45-55'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') self.assertEqual(headers['Content-Length'], '5') self.assertEqual(body, 'ddddd') def test_range_get_manifest_unsatisfiable(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=100-200'}) status, headers, body = self.call_slo(req) self.assertEqual(status, '416 Requested Range Not Satisfiable') def test_multi_range_get_manifest(self): # SLO doesn't support multi-range GETs. The way that you express # "unsupported" in HTTP is to return a 200 and the whole entity. req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-0,2-2'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '50') self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') def test_get_segment_with_non_ascii_name(self): segment_body = u"a møøse once bit my sister".encode("utf-8") self.app.register( 'GET', u'/v1/AUTH_test/ünicode/öbject-segment'.encode('utf-8'), swob.HTTPOk, {'Content-Length': str(len(segment_body)), 'Etag': md5hex(segment_body)}, segment_body) manifest_json = json.dumps([{'name': u'/ünicode/öbject-segment', 'hash': md5hex(segment_body), 'content_type': 'text/plain', 'bytes': len(segment_body)}]) self.app.register( 'GET', u'/v1/AUTH_test/ünicode/manifest'.encode('utf-8'), swob.HTTPOk, {'Content-Type': 'application/json', 'Content-Length': str(len(manifest_json)), 'X-Static-Large-Object': 'true'}, manifest_json) req = Request.blank( '/v1/AUTH_test/ünicode/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(body, segment_body) def test_get_bogus_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-badjson', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(headers['X-Object-Meta-Fish'], 'Bass') self.assertEqual(body, '') def test_head_manifest_is_efficient(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_slo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '50') self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag) self.assertEqual(body, '') # Note the lack of recursive descent into manifest-bc. We know the # content-length from the outer manifest, so there's no need for any # submanifest fetching here, but a naïve implementation might do it # anyway. self.assertEqual(self.app.calls, [ ('HEAD', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/manifest-abcd')]) def test_recursion_limit(self): # man1 points to obj1 and man2, man2 points to obj2 and man3... for i in xrange(20): self.app.register('GET', '/v1/AUTH_test/gettest/obj%d' % i, swob.HTTPOk, {'Content-Type': 'text/plain', 'Etag': md5hex('body%02d' % i)}, 'body%02d' % i) manifest_json = json.dumps([{'name': '/gettest/obj20', 'hash': md5hex('body20'), 'content_type': 'text/plain', 'bytes': '6'}]) self.app.register( 'GET', '/v1/AUTH_test/gettest/man%d' % i, swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'Etag': 'man%d' % i}, manifest_json) for i in xrange(19, 0, -1): manifest_data = [ {'name': '/gettest/obj%d' % i, 'hash': md5hex('body%02d' % i), 'bytes': '6', 'content_type': 'text/plain'}, {'name': '/gettest/man%d' % (i + 1), 'hash': 'man%d' % (i + 1), 'sub_slo': True, 'bytes': len(manifest_json), 'content_type': 'application/json;swift_bytes=%d' % ((21 - i) * 6)}] manifest_json = json.dumps(manifest_data) self.app.register( 'GET', '/v1/AUTH_test/gettest/man%d' % i, swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'Etag': 'man%d' % i}, manifest_json) req = Request.blank( '/v1/AUTH_test/gettest/man1', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertTrue(isinstance(exc, ListingIterError)) # we don't know at header-sending time that things are going to go # wrong, so we end up with a 200 and a truncated body self.assertEqual(status, '200 OK') self.assertEqual(body, ('body01body02body03body04body05' + 'body06body07body08body09body10')) # make sure we didn't keep asking for segments self.assertEqual(self.app.call_count, 20) def test_get_with_if_modified_since(self): # It's important not to pass the If-[Un]Modified-Since header to the # proxy for segment or submanifest GET requests, as it may result in # 304 Not Modified responses, and those don't contain any useful data. req = swob.Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': 'Wed, 12 Feb 2014 22:24:52 GMT', 'If-Unmodified-Since': 'Thu, 13 Feb 2014 23:25:53 GMT'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) for _, _, hdrs in self.app.calls_with_headers[1:]: self.assertFalse('If-Modified-Since' in hdrs) self.assertFalse('If-Unmodified-Since' in hdrs) def test_error_fetching_segment(self): self.app.register('GET', '/v1/AUTH_test/gettest/c_15', swob.HTTPUnauthorized, {}, None) req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertTrue(isinstance(exc, SegmentError)) self.assertEqual(status, '200 OK') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), # This one has the error, and so is the last one we fetch. ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) def test_error_fetching_submanifest(self): self.app.register('GET', '/v1/AUTH_test/gettest/manifest-bc', swob.HTTPUnauthorized, {}, None) req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) self.assertTrue(isinstance(exc, ListingIterError)) self.assertEqual("200 OK", status) self.assertEqual("aaaaa", body) self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), # This one has the error, and so is the last one we fetch. ('GET', '/v1/AUTH_test/gettest/manifest-bc')]) def test_error_fetching_first_segment_submanifest(self): # This differs from the normal submanifest error because this one # happens before we've actually sent any response body. self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-a', swob.HTTPForbidden, {}, None) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-manifest-a', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/manifest-a', 'sub_slo': True, 'content_type': 'application/json;swift_bytes=5', 'hash': 'manifest-a', 'bytes': '12345'}])) req = Request.blank( '/v1/AUTH_test/gettest/manifest-manifest-a', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) self.assertTrue(isinstance(exc, ListingIterError)) self.assertEqual('200 OK', status) self.assertEqual(body, ' ') def test_invalid_json_submanifest(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc', swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus'}, "[this {isn't (JSON") req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) self.assertTrue(isinstance(exc, ListingIterError)) self.assertEqual('200 OK', status) self.assertEqual(body, 'aaaaa') def test_mismatched_etag(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-a-b-badetag-c', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/b_10', 'hash': 'wrong!', 'content_type': 'text/plain', 'bytes': '10'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}])) req = Request.blank( '/v1/AUTH_test/gettest/manifest-a-b-badetag-c', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) self.assertTrue(isinstance(exc, SegmentError)) self.assertEqual('200 OK', status) self.assertEqual(body, 'aaaaa') def test_mismatched_size(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-a-b-badsize-c', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/a_5', 'hash': md5hex('a' * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/b_10', 'hash': md5hex('b' * 10), 'content_type': 'text/plain', 'bytes': '999999'}, {'name': '/gettest/c_15', 'hash': md5hex('c' * 15), 'content_type': 'text/plain', 'bytes': '15'}])) req = Request.blank( '/v1/AUTH_test/gettest/manifest-a-b-badsize-c', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_slo(req, expect_exception=True) self.assertTrue(isinstance(exc, SegmentError)) self.assertEqual('200 OK', status) self.assertEqual(body, 'aaaaa') def test_download_takes_too_long(self): the_time = [time.time()] def mock_time(): return the_time[0] # this is just a convenient place to hang a time jump; there's nothing # special about the choice of is_success(). def mock_is_success(status_int): the_time[0] += 7 * 3600 return status_int // 100 == 2 req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', environ={'REQUEST_METHOD': 'GET'}) with nested(patch.object(slo, 'is_success', mock_is_success), patch('swift.common.request_helpers.time.time', mock_time), patch('swift.common.request_helpers.is_success', mock_is_success)): status, headers, body, exc = self.call_slo( req, expect_exception=True) self.assertTrue(isinstance(exc, SegmentError)) self.assertEqual(status, '200 OK') self.assertEqual(self.app.calls, [ ('GET', '/v1/AUTH_test/gettest/manifest-abcd'), ('GET', '/v1/AUTH_test/gettest/a_5?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/manifest-bc'), ('GET', '/v1/AUTH_test/gettest/b_10?multipart-manifest=get'), ('GET', '/v1/AUTH_test/gettest/c_15?multipart-manifest=get')]) class TestSloBulkLogger(unittest.TestCase): def test_reused_logger(self): slo_mware = slo.filter_factory({})('fake app') self.assertTrue(slo_mware.logger is slo_mware.bulk_deleter.logger) class TestSloCopyHook(SloTestCase): def setUp(self): super(TestSloCopyHook, self).setUp() self.app.register( 'GET', '/v1/AUTH_test/c/o', swob.HTTPOk, {'Content-Length': '3', 'Etag': md5hex("obj")}, "obj") self.app.register( 'GET', '/v1/AUTH_test/c/man', swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/c/o', 'hash': md5hex("obj"), 'bytes': '3'}])) copy_hook = [None] # slip this guy in there to pull out the hook def extract_copy_hook(env, sr): copy_hook[0] = env['swift.copy_hook'] return self.app(env, sr) self.slo = slo.filter_factory({})(extract_copy_hook) req = Request.blank('/v1/AUTH_test/c/o', environ={'REQUEST_METHOD': 'GET'}) self.slo(req.environ, fake_start_response) self.copy_hook = copy_hook[0] self.assertTrue(self.copy_hook is not None) # sanity check def test_copy_hook_passthrough(self): source_req = Request.blank( '/v1/AUTH_test/c/o', environ={'REQUEST_METHOD': 'GET'}) sink_req = Request.blank( '/v1/AUTH_test/c/o', environ={'REQUEST_METHOD': 'PUT'}) # no X-Static-Large-Object header, so do nothing source_resp = Response(request=source_req, status=200) modified_resp = self.copy_hook(source_req, source_resp, sink_req) self.assertTrue(modified_resp is source_resp) def test_copy_hook_manifest(self): source_req = Request.blank( '/v1/AUTH_test/c/o', environ={'REQUEST_METHOD': 'GET'}) sink_req = Request.blank( '/v1/AUTH_test/c/o', environ={'REQUEST_METHOD': 'PUT'}) source_resp = Response(request=source_req, status=200, headers={"X-Static-Large-Object": "true"}, app_iter=[json.dumps([{'name': '/c/o', 'hash': 'obj-etag', 'bytes': '3'}])]) modified_resp = self.copy_hook(source_req, source_resp, sink_req) self.assertTrue(modified_resp is not source_resp) self.assertEqual(modified_resp.etag, md5("obj-etag").hexdigest()) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} utils._swift_admin_info = {} def test_registered_defaults(self): mware = slo.filter_factory({})('have to pass in an app') swift_info = utils.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) self.assertEqual(swift_info['slo'].get('min_segment_size'), mware.min_segment_size) self.assertEqual(swift_info['slo'].get('max_manifest_size'), mware.max_manifest_size) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_healthcheck.py0000664000175400017540000000552512323703611025313 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shutil import tempfile import unittest from swift.common.swob import Request, Response from swift.common.middleware import healthcheck class FakeApp(object): def __call__(self, env, start_response): req = Request(env) return Response(request=req, body='FAKE APP')( env, start_response) class TestHealthCheck(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() self.disable_path = os.path.join(self.tempdir, 'dont-taze-me-bro') self.got_statuses = [] def tearDown(self): shutil.rmtree(self.tempdir, ignore_errors=True) def get_app(self, app, global_conf, **local_conf): factory = healthcheck.filter_factory(global_conf, **local_conf) return factory(app) def start_response(self, status, headers): self.got_statuses.append(status) def test_healthcheck(self): req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'}) app = self.get_app(FakeApp(), {}) resp = app(req.environ, self.start_response) self.assertEquals(['200 OK'], self.got_statuses) self.assertEquals(resp, ['OK']) def test_healtcheck_pass(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) app = self.get_app(FakeApp(), {}) resp = app(req.environ, self.start_response) self.assertEquals(['200 OK'], self.got_statuses) self.assertEquals(resp, ['FAKE APP']) def test_healthcheck_pass_not_disabled(self): req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'}) app = self.get_app(FakeApp(), {}, disable_path=self.disable_path) resp = app(req.environ, self.start_response) self.assertEquals(['200 OK'], self.got_statuses) self.assertEquals(resp, ['OK']) def test_healthcheck_pass_disabled(self): open(self.disable_path, 'w') req = Request.blank('/healthcheck', environ={'REQUEST_METHOD': 'GET'}) app = self.get_app(FakeApp(), {}, disable_path=self.disable_path) resp = app(req.environ, self.start_response) self.assertEquals(['503 Service Unavailable'], self.got_statuses) self.assertEquals(resp, ['DISABLED BY FILE']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_cname_lookup.py0000664000175400017540000002172412323703611025523 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import mock from nose import SkipTest try: # this test requires the dnspython package to be installed import dns.resolver # noqa except ImportError: skip = True else: # executed if the try has no errors skip = False from swift.common.middleware import cname_lookup from swift.common.swob import Request class FakeApp(object): def __call__(self, env, start_response): return "FAKE APP" def start_response(*args): pass original_lookup = cname_lookup.lookup_cname class TestCNAMELookup(unittest.TestCase): def setUp(self): if skip: raise SkipTest self.app = cname_lookup.CNAMELookupMiddleware(FakeApp(), {'lookup_depth': 2}) def test_pass_ip_addresses(self): cname_lookup.lookup_cname = original_lookup req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': '10.134.23.198'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'fc00:7ea1:f155::6321:8841'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') def test_passthrough(self): def my_lookup(d): return 0, d cname_lookup.lookup_cname = my_lookup req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'foo.example.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'foo.example.com:8080'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'SERVER_NAME': 'foo.example.com'}, headers={'Host': None}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') def test_good_lookup(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com'}) def my_lookup(d): return 0, '%s.example.com' % d cname_lookup.lookup_cname = my_lookup resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com:8080'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'SERVER_NAME': 'mysite.com'}, headers={'Host': None}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') def test_lookup_chain_too_long(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com'}) def my_lookup(d): if d == 'mysite.com': site = 'level1.foo.com' elif d == 'level1.foo.com': site = 'level2.foo.com' elif d == 'level2.foo.com': site = 'bar.example.com' return 0, site cname_lookup.lookup_cname = my_lookup resp = self.app(req.environ, start_response) self.assertEquals(resp, ['CNAME lookup failed after 2 tries']) def test_lookup_chain_bad_target(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com'}) def my_lookup(d): return 0, 'some.invalid.site.com' cname_lookup.lookup_cname = my_lookup resp = self.app(req.environ, start_response) self.assertEquals(resp, ['CNAME lookup failed to resolve to a valid domain']) def test_something_weird(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'mysite.com'}) def my_lookup(d): return 0, None cname_lookup.lookup_cname = my_lookup resp = self.app(req.environ, start_response) self.assertEquals(resp, ['CNAME lookup failed to resolve to a valid domain']) def test_with_memcache(self): def my_lookup(d): return 0, '%s.example.com' % d cname_lookup.lookup_cname = my_lookup class memcache_stub(object): def __init__(self): self.cache = {} def get(self, key): return self.cache.get(key, None) def set(self, key, value, *a, **kw): self.cache[key] = value memcache = memcache_stub() req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', 'swift.cache': memcache}, headers={'Host': 'mysite.com'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') def test_cname_matching_ending_not_domain(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'foo.com'}) def my_lookup(d): return 0, 'c.aexample.com' cname_lookup.lookup_cname = my_lookup resp = self.app(req.environ, start_response) self.assertEquals(resp, ['CNAME lookup failed to resolve to a valid domain']) def test_cname_configured_with_empty_storage_domain(self): app = cname_lookup.CNAMELookupMiddleware(FakeApp(), {'storage_domain': '', 'lookup_depth': 2}) req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.a.example.com'}) def my_lookup(d): return 0, None cname_lookup.lookup_cname = my_lookup resp = app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') def test_storage_domains_conf_format(self): conf = {'storage_domain': 'foo.com'} app = cname_lookup.filter_factory(conf)(FakeApp()) self.assertEquals(app.storage_domain, ['.foo.com']) conf = {'storage_domain': 'foo.com, '} app = cname_lookup.filter_factory(conf)(FakeApp()) self.assertEquals(app.storage_domain, ['.foo.com']) conf = {'storage_domain': 'foo.com, bar.com'} app = cname_lookup.filter_factory(conf)(FakeApp()) self.assertEquals(app.storage_domain, ['.foo.com', '.bar.com']) conf = {'storage_domain': 'foo.com, .bar.com'} app = cname_lookup.filter_factory(conf)(FakeApp()) self.assertEquals(app.storage_domain, ['.foo.com', '.bar.com']) conf = {'storage_domain': '.foo.com, .bar.com'} app = cname_lookup.filter_factory(conf)(FakeApp()) self.assertEquals(app.storage_domain, ['.foo.com', '.bar.com']) def test_multiple_storage_domains(self): conf = {'storage_domain': 'storage1.com, storage2.com', 'lookup_depth': 2} app = cname_lookup.CNAMELookupMiddleware(FakeApp(), conf) def do_test(lookup_back): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, headers={'Host': 'c.a.example.com'}) module = 'swift.common.middleware.cname_lookup.lookup_cname' with mock.patch(module, lambda x: (0, lookup_back)): return app(req.environ, start_response) resp = do_test('c.storage1.com') self.assertEquals(resp, 'FAKE APP') resp = do_test('c.storage2.com') self.assertEquals(resp, 'FAKE APP') bad_domain = ['CNAME lookup failed to resolve to a valid domain'] resp = do_test('c.badtest.com') self.assertEquals(resp, bad_domain) swift-1.13.1/test/unit/common/middleware/test_recon.py0000664000175400017540000012170512323703614024160 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from unittest import TestCase from contextlib import contextmanager from posix import stat_result, statvfs_result import os import mock import swift.common.constraints from swift import __version__ as swiftver from swift.common.swob import Request from swift.common.middleware import recon from swift.common.utils import json def fake_check_mount(a, b): raise OSError('Input/Output Error') class FakeApp(object): def __call__(self, env, start_response): return "FAKE APP" def start_response(*args): pass class FakeFromCache(object): def __init__(self, out=None): self.fakeout = out self.fakeout_calls = [] def fake_from_recon_cache(self, *args, **kwargs): self.fakeout_calls.append((args, kwargs)) return self.fakeout class OpenAndReadTester(object): def __init__(self, output_iter): self.index = 0 self.out_len = len(output_iter) - 1 self.data = output_iter self.output_iter = iter(output_iter) self.read_calls = [] self.open_calls = [] def __iter__(self): return self def next(self): if self.index == self.out_len: raise StopIteration else: line = self.data[self.index] self.index += 1 return line def read(self, *args, **kwargs): self.read_calls.append((args, kwargs)) try: return self.output_iter.next() except StopIteration: return '' @contextmanager def open(self, *args, **kwargs): self.open_calls.append((args, kwargs)) yield self class MockOS(object): def __init__(self, ls_out=None, im_out=False, statvfs_out=None): self.ls_output = ls_out self.ismount_output = im_out self.statvfs_output = statvfs_out self.listdir_calls = [] self.statvfs_calls = [] self.ismount_calls = [] def fake_listdir(self, *args, **kwargs): self.listdir_calls.append((args, kwargs)) return self.ls_output def fake_ismount(self, *args, **kwargs): self.ismount_calls.append((args, kwargs)) if isinstance(self.ismount_output, Exception): raise self.ismount_output else: return self.ismount_output def fake_statvfs(self, *args, **kwargs): self.statvfs_calls.append((args, kwargs)) return statvfs_result(self.statvfs_output) class FakeRecon(object): def __init__(self): self.fake_replication_rtype = None self.fake_updater_rtype = None self.fake_auditor_rtype = None self.fake_expirer_rtype = None def fake_mem(self): return {'memtest': "1"} def fake_load(self): return {'loadtest': "1"} def fake_async(self): return {'asynctest': "1"} def fake_get_device_info(self): return {"/srv/1/node": ["sdb1"]} def fake_replication(self, recon_type): self.fake_replication_rtype = recon_type return {'replicationtest': "1"} def fake_updater(self, recon_type): self.fake_updater_rtype = recon_type return {'updatertest': "1"} def fake_auditor(self, recon_type): self.fake_auditor_rtype = recon_type return {'auditortest': "1"} def fake_expirer(self, recon_type): self.fake_expirer_rtype = recon_type return {'expirertest': "1"} def fake_mounted(self): return {'mountedtest': "1"} def fake_unmounted(self): return {'unmountedtest': "1"} def fake_no_unmounted(self): return [] def fake_diskusage(self): return {'diskusagetest': "1"} def fake_ringmd5(self): return {'ringmd5test': "1"} def fake_quarantined(self): return {'quarantinedtest': "1"} def fake_sockstat(self): return {'sockstattest': "1"} def nocontent(self): return None def raise_IOError(self, *args, **kwargs): raise IOError def raise_ValueError(self, *args, **kwargs): raise ValueError def raise_Exception(self, *args, **kwargs): raise Exception class TestReconSuccess(TestCase): def setUp(self): self.app = recon.ReconMiddleware(FakeApp(), {}) self.mockos = MockOS() self.fakecache = FakeFromCache() self.real_listdir = os.listdir self.real_ismount = swift.common.constraints.ismount self.real_statvfs = os.statvfs os.listdir = self.mockos.fake_listdir swift.common.constraints.ismount = self.mockos.fake_ismount os.statvfs = self.mockos.fake_statvfs self.real_from_cache = self.app._from_recon_cache self.app._from_recon_cache = self.fakecache.fake_from_recon_cache self.frecon = FakeRecon() def tearDown(self): os.listdir = self.real_listdir swift.common.constraints.ismount = self.real_ismount os.statvfs = self.real_statvfs del self.mockos self.app._from_recon_cache = self.real_from_cache del self.fakecache def test_from_recon_cache(self): oart = OpenAndReadTester(['{"notneeded": 5, "testkey1": "canhazio"}']) self.app._from_recon_cache = self.real_from_cache rv = self.app._from_recon_cache(['testkey1', 'notpresentkey'], 'test.cache', openr=oart.open) self.assertEquals(oart.read_calls, [((), {})]) self.assertEquals(oart.open_calls, [(('test.cache', 'r'), {})]) self.assertEquals(rv, {'notpresentkey': None, 'testkey1': 'canhazio'}) self.app._from_recon_cache = self.fakecache.fake_from_recon_cache def test_from_recon_cache_ioerror(self): oart = self.frecon.raise_IOError self.app._from_recon_cache = self.real_from_cache rv = self.app._from_recon_cache(['testkey1', 'notpresentkey'], 'test.cache', openr=oart) self.assertEquals(rv, {'notpresentkey': None, 'testkey1': None}) self.app._from_recon_cache = self.fakecache.fake_from_recon_cache def test_from_recon_cache_valueerror(self): oart = self.frecon.raise_ValueError self.app._from_recon_cache = self.real_from_cache rv = self.app._from_recon_cache(['testkey1', 'notpresentkey'], 'test.cache', openr=oart) self.assertEquals(rv, {'notpresentkey': None, 'testkey1': None}) self.app._from_recon_cache = self.fakecache.fake_from_recon_cache def test_from_recon_cache_exception(self): oart = self.frecon.raise_Exception self.app._from_recon_cache = self.real_from_cache rv = self.app._from_recon_cache(['testkey1', 'notpresentkey'], 'test.cache', openr=oart) self.assertEquals(rv, {'notpresentkey': None, 'testkey1': None}) self.app._from_recon_cache = self.fakecache.fake_from_recon_cache def test_get_mounted(self): mounts_content = [ 'rootfs / rootfs rw 0 0', 'none /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0', 'none /proc proc rw,nosuid,nodev,noexec,relatime 0 0', 'none /dev devtmpfs rw,relatime,size=248404k,nr_inodes=62101,' 'mode=755 0 0', 'none /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,' 'ptmxmode=000 0 0', '/dev/disk/by-uuid/e5b143bd-9f31-49a7-b018-5e037dc59252 / ext4' ' rw,relatime,errors=remount-ro,barrier=1,data=ordered 0 0', 'none /sys/fs/fuse/connections fusectl rw,relatime 0 0', 'none /sys/kernel/debug debugfs rw,relatime 0 0', 'none /sys/kernel/security securityfs rw,relatime 0 0', 'none /dev/shm tmpfs rw,nosuid,nodev,relatime 0 0', 'none /var/run tmpfs rw,nosuid,relatime,mode=755 0 0', 'none /var/lock tmpfs rw,nosuid,nodev,noexec,relatime 0 0', 'none /lib/init/rw tmpfs rw,nosuid,relatime,mode=755 0 0', '/dev/loop0 /mnt/sdb1 xfs rw,noatime,nodiratime,attr2,nobarrier,' 'logbufs=8,noquota 0 0', 'rpc_pipefs /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0', 'nfsd /proc/fs/nfsd nfsd rw,relatime 0 0', 'none /proc/fs/vmblock/mountPoint vmblock rw,relatime 0 0', ''] mounted_resp = [ {'device': 'rootfs', 'path': '/'}, {'device': 'none', 'path': '/sys'}, {'device': 'none', 'path': '/proc'}, {'device': 'none', 'path': '/dev'}, {'device': 'none', 'path': '/dev/pts'}, {'device': '/dev/disk/by-uuid/' 'e5b143bd-9f31-49a7-b018-5e037dc59252', 'path': '/'}, {'device': 'none', 'path': '/sys/fs/fuse/connections'}, {'device': 'none', 'path': '/sys/kernel/debug'}, {'device': 'none', 'path': '/sys/kernel/security'}, {'device': 'none', 'path': '/dev/shm'}, {'device': 'none', 'path': '/var/run'}, {'device': 'none', 'path': '/var/lock'}, {'device': 'none', 'path': '/lib/init/rw'}, {'device': '/dev/loop0', 'path': '/mnt/sdb1'}, {'device': 'rpc_pipefs', 'path': '/var/lib/nfs/rpc_pipefs'}, {'device': 'nfsd', 'path': '/proc/fs/nfsd'}, {'device': 'none', 'path': '/proc/fs/vmblock/mountPoint'}] oart = OpenAndReadTester(mounts_content) rv = self.app.get_mounted(openr=oart.open) self.assertEquals(oart.open_calls, [(('/proc/mounts', 'r'), {})]) self.assertEquals(rv, mounted_resp) def test_get_load(self): oart = OpenAndReadTester(['0.03 0.03 0.00 1/220 16306']) rv = self.app.get_load(openr=oart.open) self.assertEquals(oart.read_calls, [((), {})]) self.assertEquals(oart.open_calls, [(('/proc/loadavg', 'r'), {})]) self.assertEquals(rv, {'5m': 0.029999999999999999, '15m': 0.0, 'processes': 16306, 'tasks': '1/220', '1m': 0.029999999999999999}) def test_get_mem(self): meminfo_content = ['MemTotal: 505840 kB', 'MemFree: 26588 kB', 'Buffers: 44948 kB', 'Cached: 146376 kB', 'SwapCached: 14736 kB', 'Active: 194900 kB', 'Inactive: 193412 kB', 'Active(anon): 94208 kB', 'Inactive(anon): 102848 kB', 'Active(file): 100692 kB', 'Inactive(file): 90564 kB', 'Unevictable: 0 kB', 'Mlocked: 0 kB', 'SwapTotal: 407544 kB', 'SwapFree: 313436 kB', 'Dirty: 104 kB', 'Writeback: 0 kB', 'AnonPages: 185268 kB', 'Mapped: 9592 kB', 'Shmem: 68 kB', 'Slab: 61716 kB', 'SReclaimable: 46620 kB', 'SUnreclaim: 15096 kB', 'KernelStack: 1760 kB', 'PageTables: 8832 kB', 'NFS_Unstable: 0 kB', 'Bounce: 0 kB', 'WritebackTmp: 0 kB', 'CommitLimit: 660464 kB', 'Committed_AS: 565608 kB', 'VmallocTotal: 34359738367 kB', 'VmallocUsed: 266724 kB', 'VmallocChunk: 34359467156 kB', 'HardwareCorrupted: 0 kB', 'HugePages_Total: 0', 'HugePages_Free: 0', 'HugePages_Rsvd: 0', 'HugePages_Surp: 0', 'Hugepagesize: 2048 kB', 'DirectMap4k: 10240 kB', 'DirectMap2M: 514048 kB', ''] meminfo_resp = {'WritebackTmp': '0 kB', 'SwapTotal': '407544 kB', 'Active(anon)': '94208 kB', 'SwapFree': '313436 kB', 'DirectMap4k': '10240 kB', 'KernelStack': '1760 kB', 'MemFree': '26588 kB', 'HugePages_Rsvd': '0', 'Committed_AS': '565608 kB', 'Active(file)': '100692 kB', 'NFS_Unstable': '0 kB', 'VmallocChunk': '34359467156 kB', 'Writeback': '0 kB', 'Inactive(file)': '90564 kB', 'MemTotal': '505840 kB', 'VmallocUsed': '266724 kB', 'HugePages_Free': '0', 'AnonPages': '185268 kB', 'Active': '194900 kB', 'Inactive(anon)': '102848 kB', 'CommitLimit': '660464 kB', 'Hugepagesize': '2048 kB', 'Cached': '146376 kB', 'SwapCached': '14736 kB', 'VmallocTotal': '34359738367 kB', 'Shmem': '68 kB', 'Mapped': '9592 kB', 'SUnreclaim': '15096 kB', 'Unevictable': '0 kB', 'SReclaimable': '46620 kB', 'Mlocked': '0 kB', 'DirectMap2M': '514048 kB', 'HugePages_Surp': '0', 'Bounce': '0 kB', 'Inactive': '193412 kB', 'PageTables': '8832 kB', 'HardwareCorrupted': '0 kB', 'HugePages_Total': '0', 'Slab': '61716 kB', 'Buffers': '44948 kB', 'Dirty': '104 kB'} oart = OpenAndReadTester(meminfo_content) rv = self.app.get_mem(openr=oart.open) self.assertEquals(oart.open_calls, [(('/proc/meminfo', 'r'), {})]) self.assertEquals(rv, meminfo_resp) def test_get_async_info(self): from_cache_response = {'async_pending': 5} self.fakecache.fakeout = from_cache_response rv = self.app.get_async_info() self.assertEquals(rv, {'async_pending': 5}) def test_get_replication_info_account(self): from_cache_response = { "replication_stats": { "attempted": 1, "diff": 0, "diff_capped": 0, "empty": 0, "failure": 0, "hashmatch": 0, "no_change": 2, "remote_merge": 0, "remove": 0, "rsync": 0, "start": 1333044050.855202, "success": 2, "ts_repl": 0}, "replication_time": 0.2615511417388916, "replication_last": 1357969645.25} self.fakecache.fakeout = from_cache_response rv = self.app.get_replication_info('account') self.assertEquals(self.fakecache.fakeout_calls, [((['replication_time', 'replication_stats', 'replication_last'], '/var/cache/swift/account.recon'), {})]) self.assertEquals(rv, { "replication_stats": { "attempted": 1, "diff": 0, "diff_capped": 0, "empty": 0, "failure": 0, "hashmatch": 0, "no_change": 2, "remote_merge": 0, "remove": 0, "rsync": 0, "start": 1333044050.855202, "success": 2, "ts_repl": 0}, "replication_time": 0.2615511417388916, "replication_last": 1357969645.25}) def test_get_replication_info_container(self): from_cache_response = { "replication_time": 200.0, "replication_stats": { "attempted": 179, "diff": 0, "diff_capped": 0, "empty": 0, "failure": 0, "hashmatch": 0, "no_change": 358, "remote_merge": 0, "remove": 0, "rsync": 0, "start": 5.5, "success": 358, "ts_repl": 0}, "replication_last": 1357969645.25} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_replication_info('container') self.assertEquals(self.fakecache.fakeout_calls, [((['replication_time', 'replication_stats', 'replication_last'], '/var/cache/swift/container.recon'), {})]) self.assertEquals(rv, { "replication_time": 200.0, "replication_stats": { "attempted": 179, "diff": 0, "diff_capped": 0, "empty": 0, "failure": 0, "hashmatch": 0, "no_change": 358, "remote_merge": 0, "remove": 0, "rsync": 0, "start": 5.5, "success": 358, "ts_repl": 0}, "replication_last": 1357969645.25}) def test_get_replication_object(self): from_cache_response = {"object_replication_time": 200.0, "object_replication_last": 1357962809.15} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_replication_info('object') self.assertEquals(self.fakecache.fakeout_calls, [((['object_replication_time', 'object_replication_last'], '/var/cache/swift/object.recon'), {})]) self.assertEquals(rv, {'object_replication_time': 200.0, 'object_replication_last': 1357962809.15}) def test_get_updater_info_container(self): from_cache_response = {"container_updater_sweep": 18.476239919662476} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_updater_info('container') self.assertEquals(self.fakecache.fakeout_calls, [((['container_updater_sweep'], '/var/cache/swift/container.recon'), {})]) self.assertEquals(rv, {"container_updater_sweep": 18.476239919662476}) def test_get_updater_info_object(self): from_cache_response = {"object_updater_sweep": 0.79848217964172363} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_updater_info('object') self.assertEquals(self.fakecache.fakeout_calls, [((['object_updater_sweep'], '/var/cache/swift/object.recon'), {})]) self.assertEquals(rv, {"object_updater_sweep": 0.79848217964172363}) def test_get_auditor_info_account(self): from_cache_response = {"account_auditor_pass_completed": 0.24, "account_audits_failed": 0, "account_audits_passed": 6, "account_audits_since": "1333145374.1373529"} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_auditor_info('account') self.assertEquals(self.fakecache.fakeout_calls, [((['account_audits_passed', 'account_auditor_pass_completed', 'account_audits_since', 'account_audits_failed'], '/var/cache/swift/account.recon'), {})]) self.assertEquals(rv, {"account_auditor_pass_completed": 0.24, "account_audits_failed": 0, "account_audits_passed": 6, "account_audits_since": "1333145374.1373529"}) def test_get_auditor_info_container(self): from_cache_response = {"container_auditor_pass_completed": 0.24, "container_audits_failed": 0, "container_audits_passed": 6, "container_audits_since": "1333145374.1373529"} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_auditor_info('container') self.assertEquals(self.fakecache.fakeout_calls, [((['container_audits_passed', 'container_auditor_pass_completed', 'container_audits_since', 'container_audits_failed'], '/var/cache/swift/container.recon'), {})]) self.assertEquals(rv, {"container_auditor_pass_completed": 0.24, "container_audits_failed": 0, "container_audits_passed": 6, "container_audits_since": "1333145374.1373529"}) def test_get_auditor_info_object(self): from_cache_response = { "object_auditor_stats_ALL": { "audit_time": 115.14418768882751, "bytes_processed": 234660, "completed": 115.4512460231781, "errors": 0, "files_processed": 2310, "quarantined": 0}, "object_auditor_stats_ZBF": { "audit_time": 45.877294063568115, "bytes_processed": 0, "completed": 46.181446075439453, "errors": 0, "files_processed": 2310, "quarantined": 0}} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_auditor_info('object') self.assertEquals(self.fakecache.fakeout_calls, [((['object_auditor_stats_ALL', 'object_auditor_stats_ZBF'], '/var/cache/swift/object.recon'), {})]) self.assertEquals(rv, { "object_auditor_stats_ALL": { "audit_time": 115.14418768882751, "bytes_processed": 234660, "completed": 115.4512460231781, "errors": 0, "files_processed": 2310, "quarantined": 0}, "object_auditor_stats_ZBF": { "audit_time": 45.877294063568115, "bytes_processed": 0, "completed": 46.181446075439453, "errors": 0, "files_processed": 2310, "quarantined": 0}}) def test_get_auditor_info_object_once(self): from_cache_response = { "object_auditor_stats_ALL": {'disk1disk2': { "audit_time": 115.14418768882751, "bytes_processed": 234660, "completed": 115.4512460231781, "errors": 0, "files_processed": 2310, "quarantined": 0}}, "object_auditor_stats_ZBF": {'disk1disk2': { "audit_time": 45.877294063568115, "bytes_processed": 0, "completed": 46.181446075439453, "errors": 0, "files_processed": 2310, "quarantined": 0}}} self.fakecache.fakeout_calls = [] self.fakecache.fakeout = from_cache_response rv = self.app.get_auditor_info('object') self.assertEquals(self.fakecache.fakeout_calls, [((['object_auditor_stats_ALL', 'object_auditor_stats_ZBF'], '/var/cache/swift/object.recon'), {})]) self.assertEquals(rv, { "object_auditor_stats_ALL": {'disk1disk2': { "audit_time": 115.14418768882751, "bytes_processed": 234660, "completed": 115.4512460231781, "errors": 0, "files_processed": 2310, "quarantined": 0}}, "object_auditor_stats_ZBF": {'disk1disk2': { "audit_time": 45.877294063568115, "bytes_processed": 0, "completed": 46.181446075439453, "errors": 0, "files_processed": 2310, "quarantined": 0}}}) def test_get_unmounted(self): unmounted_resp = [{'device': 'fakeone', 'mounted': False}, {'device': 'faketwo', 'mounted': False}] self.mockos.ls_output = ['fakeone', 'faketwo'] self.mockos.ismount_output = False rv = self.app.get_unmounted() self.assertEquals(self.mockos.listdir_calls, [(('/srv/node',), {})]) self.assertEquals(rv, unmounted_resp) def test_get_unmounted_everything_normal(self): unmounted_resp = [] self.mockos.ls_output = ['fakeone', 'faketwo'] self.mockos.ismount_output = True rv = self.app.get_unmounted() self.assertEquals(self.mockos.listdir_calls, [(('/srv/node',), {})]) self.assertEquals(rv, unmounted_resp) def test_get_unmounted_checkmount_fail(self): unmounted_resp = [{'device': 'fakeone', 'mounted': 'brokendrive'}] self.mockos.ls_output = ['fakeone'] self.mockos.ismount_output = OSError('brokendrive') rv = self.app.get_unmounted() self.assertEquals(self.mockos.listdir_calls, [(('/srv/node',), {})]) self.assertEquals(self.mockos.ismount_calls, [(('/srv/node/fakeone',), {})]) self.assertEquals(rv, unmounted_resp) def test_no_get_unmounted(self): def fake_checkmount_true(*args): return True unmounted_resp = [] self.mockos.ls_output = [] self.mockos.ismount_output = False rv = self.app.get_unmounted() self.assertEquals(self.mockos.listdir_calls, [(('/srv/node',), {})]) self.assertEquals(rv, unmounted_resp) def test_get_diskusage(self): #posix.statvfs_result(f_bsize=4096, f_frsize=4096, f_blocks=1963185, # f_bfree=1113075, f_bavail=1013351, # f_files=498736, # f_ffree=397839, f_favail=397839, f_flag=0, # f_namemax=255) statvfs_content = (4096, 4096, 1963185, 1113075, 1013351, 498736, 397839, 397839, 0, 255) du_resp = [{'device': 'canhazdrive1', 'avail': 4150685696, 'mounted': True, 'used': 3890520064, 'size': 8041205760}] self.mockos.ls_output = ['canhazdrive1'] self.mockos.statvfs_output = statvfs_content self.mockos.ismount_output = True rv = self.app.get_diskusage() self.assertEquals(self.mockos.statvfs_calls, [(('/srv/node/canhazdrive1',), {})]) self.assertEquals(rv, du_resp) def test_get_diskusage_checkmount_fail(self): du_resp = [{'device': 'canhazdrive1', 'avail': '', 'mounted': 'brokendrive', 'used': '', 'size': ''}] self.mockos.ls_output = ['canhazdrive1'] self.mockos.ismount_output = OSError('brokendrive') rv = self.app.get_diskusage() self.assertEquals(self.mockos.listdir_calls, [(('/srv/node',), {})]) self.assertEquals(self.mockos.ismount_calls, [(('/srv/node/canhazdrive1',), {})]) self.assertEquals(rv, du_resp) @mock.patch("swift.common.middleware.recon.check_mount", fake_check_mount) def test_get_diskusage_oserror(self): du_resp = [{'device': 'canhazdrive1', 'avail': '', 'mounted': 'Input/Output Error', 'used': '', 'size': ''}] self.mockos.ls_output = ['canhazdrive1'] rv = self.app.get_diskusage() self.assertEquals(rv, du_resp) def test_get_quarantine_count(self): self.mockos.ls_output = ['sda'] self.mockos.ismount_output = True def fake_lstat(*args, **kwargs): #posix.lstat_result(st_mode=1, st_ino=2, st_dev=3, st_nlink=4, # st_uid=5, st_gid=6, st_size=7, st_atime=8, # st_mtime=9, st_ctime=10) return stat_result((1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) def fake_exists(*args, **kwargs): return True with mock.patch("os.lstat", fake_lstat): with mock.patch("os.path.exists", fake_exists): rv = self.app.get_quarantine_count() self.assertEquals(rv, {'objects': 2, 'accounts': 2, 'containers': 2}) def test_get_socket_info(self): sockstat_content = ['sockets: used 271', 'TCP: inuse 30 orphan 0 tw 0 alloc 31 mem 0', 'UDP: inuse 16 mem 4', 'UDPLITE: inuse 0', 'RAW: inuse 0', 'FRAG: inuse 0 memory 0', ''] oart = OpenAndReadTester(sockstat_content) self.app.get_socket_info(openr=oart.open) self.assertEquals(oart.open_calls, [ (('/proc/net/sockstat', 'r'), {}), (('/proc/net/sockstat6', 'r'), {})]) class TestReconMiddleware(unittest.TestCase): def setUp(self): self.frecon = FakeRecon() self.app = recon.ReconMiddleware(FakeApp(), {'object_recon': "true"}) #self.app.object_recon = True self.app.get_mem = self.frecon.fake_mem self.app.get_load = self.frecon.fake_load self.app.get_async_info = self.frecon.fake_async self.app.get_device_info = self.frecon.fake_get_device_info self.app.get_replication_info = self.frecon.fake_replication self.app.get_auditor_info = self.frecon.fake_auditor self.app.get_updater_info = self.frecon.fake_updater self.app.get_expirer_info = self.frecon.fake_expirer self.app.get_mounted = self.frecon.fake_mounted self.app.get_unmounted = self.frecon.fake_unmounted self.app.get_diskusage = self.frecon.fake_diskusage self.app.get_ring_md5 = self.frecon.fake_ringmd5 self.app.get_quarantine_count = self.frecon.fake_quarantined self.app.get_socket_info = self.frecon.fake_sockstat def test_recon_get_mem(self): get_mem_resp = ['{"memtest": "1"}'] req = Request.blank('/recon/mem', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_mem_resp) def test_recon_get_version(self): req = Request.blank('/recon/version', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, [json.dumps({'version': swiftver})]) def test_recon_get_load(self): get_load_resp = ['{"loadtest": "1"}'] req = Request.blank('/recon/load', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_load_resp) def test_recon_get_async(self): get_async_resp = ['{"asynctest": "1"}'] req = Request.blank('/recon/async', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_async_resp) def test_get_device_info(self): get_device_resp = ['{"/srv/1/node": ["sdb1"]}'] req = Request.blank('/recon/devices', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_device_resp) def test_recon_get_replication_notype(self): get_replication_resp = ['{"replicationtest": "1"}'] req = Request.blank('/recon/replication', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_replication_resp) self.assertEquals(self.frecon.fake_replication_rtype, 'object') self.frecon.fake_replication_rtype = None def test_recon_get_replication_all(self): get_replication_resp = ['{"replicationtest": "1"}'] #test account req = Request.blank('/recon/replication/account', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_replication_resp) self.assertEquals(self.frecon.fake_replication_rtype, 'account') self.frecon.fake_replication_rtype = None #test container req = Request.blank('/recon/replication/container', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_replication_resp) self.assertEquals(self.frecon.fake_replication_rtype, 'container') self.frecon.fake_replication_rtype = None #test object req = Request.blank('/recon/replication/object', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_replication_resp) self.assertEquals(self.frecon.fake_replication_rtype, 'object') self.frecon.fake_replication_rtype = None def test_recon_get_auditor_invalid(self): get_auditor_resp = ['Invalid path: /recon/auditor/invalid'] req = Request.blank('/recon/auditor/invalid', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_auditor_resp) def test_recon_get_auditor_notype(self): get_auditor_resp = ['Invalid path: /recon/auditor'] req = Request.blank('/recon/auditor', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_auditor_resp) def test_recon_get_auditor_all(self): get_auditor_resp = ['{"auditortest": "1"}'] req = Request.blank('/recon/auditor/account', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_auditor_resp) self.assertEquals(self.frecon.fake_auditor_rtype, 'account') self.frecon.fake_auditor_rtype = None req = Request.blank('/recon/auditor/container', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_auditor_resp) self.assertEquals(self.frecon.fake_auditor_rtype, 'container') self.frecon.fake_auditor_rtype = None req = Request.blank('/recon/auditor/object', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_auditor_resp) self.assertEquals(self.frecon.fake_auditor_rtype, 'object') self.frecon.fake_auditor_rtype = None def test_recon_get_updater_invalid(self): get_updater_resp = ['Invalid path: /recon/updater/invalid'] req = Request.blank('/recon/updater/invalid', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_updater_resp) def test_recon_get_updater_notype(self): get_updater_resp = ['Invalid path: /recon/updater'] req = Request.blank('/recon/updater', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_updater_resp) def test_recon_get_updater(self): get_updater_resp = ['{"updatertest": "1"}'] req = Request.blank('/recon/updater/container', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(self.frecon.fake_updater_rtype, 'container') self.frecon.fake_updater_rtype = None self.assertEquals(resp, get_updater_resp) req = Request.blank('/recon/updater/object', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_updater_resp) self.assertEquals(self.frecon.fake_updater_rtype, 'object') self.frecon.fake_updater_rtype = None def test_recon_get_expirer_invalid(self): get_updater_resp = ['Invalid path: /recon/expirer/invalid'] req = Request.blank('/recon/expirer/invalid', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_updater_resp) def test_recon_get_expirer_notype(self): get_updater_resp = ['Invalid path: /recon/expirer'] req = Request.blank('/recon/expirer', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_updater_resp) def test_recon_get_expirer_object(self): get_expirer_resp = ['{"expirertest": "1"}'] req = Request.blank('/recon/expirer/object', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_expirer_resp) self.assertEquals(self.frecon.fake_expirer_rtype, 'object') self.frecon.fake_updater_rtype = None def test_recon_get_mounted(self): get_mounted_resp = ['{"mountedtest": "1"}'] req = Request.blank('/recon/mounted', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_mounted_resp) def test_recon_get_unmounted(self): get_unmounted_resp = ['{"unmountedtest": "1"}'] self.app.get_unmounted = self.frecon.fake_unmounted req = Request.blank('/recon/unmounted', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_unmounted_resp) def test_recon_no_get_unmounted(self): get_unmounted_resp = '[]' self.app.get_unmounted = self.frecon.fake_no_unmounted req = Request.blank('/recon/unmounted', environ={'REQUEST_METHOD': 'GET'}) resp = ''.join(self.app(req.environ, start_response)) self.assertEquals(resp, get_unmounted_resp) def test_recon_get_diskusage(self): get_diskusage_resp = ['{"diskusagetest": "1"}'] req = Request.blank('/recon/diskusage', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_diskusage_resp) def test_recon_get_ringmd5(self): get_ringmd5_resp = ['{"ringmd5test": "1"}'] req = Request.blank('/recon/ringmd5', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_ringmd5_resp) def test_recon_get_quarantined(self): get_quarantined_resp = ['{"quarantinedtest": "1"}'] req = Request.blank('/recon/quarantined', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_quarantined_resp) def test_recon_get_sockstat(self): get_sockstat_resp = ['{"sockstattest": "1"}'] req = Request.blank('/recon/sockstat', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, get_sockstat_resp) def test_recon_invalid_path(self): req = Request.blank('/recon/invalid', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, ['Invalid path: /recon/invalid']) def test_no_content(self): self.app.get_load = self.frecon.nocontent req = Request.blank('/recon/load', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, ['Internal server error.']) def test_recon_pass(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_bulk.py0000664000175400017540000011171112323703614024003 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import numbers import unittest import os import tarfile import urllib import zlib import mock from shutil import rmtree from tempfile import mkdtemp from StringIO import StringIO from eventlet import sleep from mock import patch, call from swift.common import utils from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED from swift.common.utils import json class FakeApp(object): def __init__(self): self.calls = 0 self.delete_paths = [] self.max_pathlen = 100 self.del_cont_total_calls = 2 self.del_cont_cur_call = 0 def __call__(self, env, start_response): self.calls += 1 if env['PATH_INFO'].startswith('/unauth/'): if env['PATH_INFO'].endswith('/c/f_ok'): return Response(status='204 No Content')(env, start_response) return Response(status=401)(env, start_response) if env['PATH_INFO'].startswith('/create_cont/'): if env['REQUEST_METHOD'] == 'HEAD': return Response(status='404 Not Found')(env, start_response) return Response(status='201 Created')(env, start_response) if env['PATH_INFO'].startswith('/create_cont_fail/'): if env['REQUEST_METHOD'] == 'HEAD': return Response(status='403 Forbidden')(env, start_response) return Response(status='404 Not Found')(env, start_response) if env['PATH_INFO'].startswith('/create_obj_unauth/'): if env['PATH_INFO'].endswith('/cont'): return Response(status='201 Created')(env, start_response) return Response(status=401)(env, start_response) if env['PATH_INFO'].startswith('/tar_works/'): if len(env['PATH_INFO']) > self.max_pathlen: return Response(status='400 Bad Request')(env, start_response) return Response(status='201 Created')(env, start_response) if env['PATH_INFO'].startswith('/tar_works_cont_head_fail/'): if env['REQUEST_METHOD'] == 'HEAD': return Response(status='404 Not Found')(env, start_response) if len(env['PATH_INFO']) > 100: return Response(status='400 Bad Request')(env, start_response) return Response(status='201 Created')(env, start_response) if (env['PATH_INFO'].startswith('/delete_works/') and env['REQUEST_METHOD'] == 'DELETE'): self.delete_paths.append(env['PATH_INFO']) if len(env['PATH_INFO']) > self.max_pathlen: return Response(status='400 Bad Request')(env, start_response) if env['PATH_INFO'].endswith('404'): return Response(status='404 Not Found')(env, start_response) if env['PATH_INFO'].endswith('badutf8'): return Response( status='412 Precondition Failed')(env, start_response) return Response(status='204 No Content')(env, start_response) if env['PATH_INFO'].startswith('/delete_cont_fail/'): return Response(status='409 Conflict')(env, start_response) if env['PATH_INFO'].startswith('/broke/'): return Response(status='500 Internal Error')(env, start_response) if env['PATH_INFO'].startswith('/delete_cont_success_after_attempts/'): if self.del_cont_cur_call < self.del_cont_total_calls: self.del_cont_cur_call += 1 return Response(status='409 Conflict')(env, start_response) else: return Response(status='204 No Content')(env, start_response) def build_dir_tree(start_path, tree_obj): if isinstance(tree_obj, list): for obj in tree_obj: build_dir_tree(start_path, obj) if isinstance(tree_obj, dict): for dir_name, obj in tree_obj.iteritems(): dir_path = os.path.join(start_path, dir_name) os.mkdir(dir_path) build_dir_tree(dir_path, obj) if isinstance(tree_obj, unicode): tree_obj = tree_obj.encode('utf8') if isinstance(tree_obj, str): obj_path = os.path.join(start_path, tree_obj) with open(obj_path, 'w+') as tree_file: tree_file.write('testing') def build_tar_tree(tar, start_path, tree_obj, base_path=''): if isinstance(tree_obj, list): for obj in tree_obj: build_tar_tree(tar, start_path, obj, base_path=base_path) if isinstance(tree_obj, dict): for dir_name, obj in tree_obj.iteritems(): dir_path = os.path.join(start_path, dir_name) tar_info = tarfile.TarInfo(dir_path[len(base_path):]) tar_info.type = tarfile.DIRTYPE tar.addfile(tar_info) build_tar_tree(tar, dir_path, obj, base_path=base_path) if isinstance(tree_obj, unicode): tree_obj = tree_obj.encode('utf8') if isinstance(tree_obj, str): obj_path = os.path.join(start_path, tree_obj) tar_info = tarfile.TarInfo('./' + obj_path[len(base_path):]) tar.addfile(tar_info) class TestUntar(unittest.TestCase): def setUp(self): self.app = FakeApp() self.bulk = bulk.filter_factory({})(self.app) self.testdir = mkdtemp(suffix='tmp_test_bulk') def tearDown(self): self.app.calls = 0 rmtree(self.testdir, ignore_errors=1) def handle_extract_and_iter(self, req, compress_format, out_content_type='application/json'): resp_body = ''.join( self.bulk.handle_extract_iter(req, compress_format, out_content_type=out_content_type)) return resp_body def test_create_container_for_path(self): req = Request.blank('/') self.assertEquals( self.bulk.create_container(req, '/create_cont/acc/cont'), True) self.assertEquals(self.app.calls, 2) self.assertRaises( bulk.CreateContainerError, self.bulk.create_container, req, '/create_cont_fail/acc/cont') self.assertEquals(self.app.calls, 3) def test_extract_tar_works(self): # On systems where $TMPDIR is long (like OS X), we need to do this # or else every upload will fail due to the path being too long. self.app.max_pathlen += len(self.testdir) for compress_format in ['', 'gz', 'bz2']: base_name = 'base_works_%s' % compress_format dir_tree = [ {base_name: [{'sub_dir1': ['sub1_file1', 'sub1_file2']}, {'sub_dir2': ['sub2_file1', u'test obj \u2661']}, 'sub_file1', {'sub_dir3': [{'sub4_dir1': '../sub4 file1'}]}, {'sub_dir4': None}, ]}] build_dir_tree(self.testdir, dir_tree) mode = 'w' extension = '' if compress_format: mode += ':' + compress_format extension += '.' + compress_format tar = tarfile.open(name=os.path.join(self.testdir, 'tar_works.tar' + extension), mode=mode) tar.add(os.path.join(self.testdir, base_name)) tar.close() req = Request.blank('/tar_works/acc/cont/') req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, compress_format) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Files Created'], 6) # test out xml req = Request.blank('/tar_works/acc/cont/') req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter( req, compress_format, 'application/xml') self.assert_('201 Created' in resp_body) self.assert_('6' in resp_body) # test out nonexistent format req = Request.blank('/tar_works/acc/cont/?extract-archive=tar', headers={'Accept': 'good_xml'}) req.environ['REQUEST_METHOD'] = 'PUT' req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar' + extension)) req.headers['transfer-encoding'] = 'chunked' def fake_start_response(*args, **kwargs): pass app_iter = self.bulk(req.environ, fake_start_response) resp_body = ''.join([i for i in app_iter]) self.assert_('Response Status: 406' in resp_body) def test_extract_call(self): base_name = 'base_works_gz' dir_tree = [ {base_name: [{'sub_dir1': ['sub1_file1', 'sub1_file2']}, {'sub_dir2': ['sub2_file1', 'sub2_file2']}, 'sub_file1', {'sub_dir3': [{'sub4_dir1': 'sub4_file1'}]}]}] build_dir_tree(self.testdir, dir_tree) tar = tarfile.open(name=os.path.join(self.testdir, 'tar_works.tar.gz'), mode='w:gz') tar.add(os.path.join(self.testdir, base_name)) tar.close() def fake_start_response(*args, **kwargs): pass req = Request.blank('/tar_works/acc/cont/?extract-archive=tar.gz') req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar.gz')) self.bulk(req.environ, fake_start_response) self.assertEquals(self.app.calls, 1) self.app.calls = 0 req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar.gz')) req.headers['transfer-encoding'] = 'Chunked' req.method = 'PUT' app_iter = self.bulk(req.environ, fake_start_response) list(app_iter) # iter over resp self.assertEquals(self.app.calls, 7) self.app.calls = 0 req = Request.blank('/tar_works/acc/cont/?extract-archive=bad') req.method = 'PUT' req.headers['transfer-encoding'] = 'Chunked' req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar.gz')) t = self.bulk(req.environ, fake_start_response) self.assertEquals(t[0], "Unsupported archive format") tar = tarfile.open(name=os.path.join(self.testdir, 'tar_works.tar'), mode='w') tar.add(os.path.join(self.testdir, base_name)) tar.close() self.app.calls = 0 req = Request.blank('/tar_works/acc/cont/?extract-archive=tar') req.method = 'PUT' req.headers['transfer-encoding'] = 'Chunked' req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar')) app_iter = self.bulk(req.environ, fake_start_response) list(app_iter) # iter over resp self.assertEquals(self.app.calls, 7) def test_bad_container(self): req = Request.blank('/invalid/', body='') resp_body = self.handle_extract_and_iter(req, '') self.assertTrue('404 Not Found' in resp_body) def test_content_length_required(self): req = Request.blank('/create_cont_fail/acc/cont') resp_body = self.handle_extract_and_iter(req, '') self.assertTrue('411 Length Required' in resp_body) def test_bad_tar(self): req = Request.blank('/create_cont_fail/acc/cont', body='') def bad_open(*args, **kwargs): raise zlib.error('bad tar') with patch.object(tarfile, 'open', bad_open): resp_body = self.handle_extract_and_iter(req, '') self.assertTrue('400 Bad Request' in resp_body) def build_tar(self, dir_tree=None): if not dir_tree: dir_tree = [ {'base_fails1': [{'sub_dir1': ['sub1_file1']}, {'sub_dir2': ['sub2_file1', 'sub2_file2']}, 'f' * 101, {'sub_dir3': [{'sub4_dir1': 'sub4_file1'}]}]}] tar = tarfile.open(name=os.path.join(self.testdir, 'tar_fails.tar'), mode='w') build_tar_tree(tar, self.testdir, dir_tree, base_path=self.testdir + '/') tar.close() return tar def test_extract_tar_with_basefile(self): dir_tree = [ 'base_lvl_file', 'another_base_file', {'base_fails1': [{'sub_dir1': ['sub1_file1']}, {'sub_dir2': ['sub2_file1', 'sub2_file2']}, {'sub_dir3': [{'sub4_dir1': 'sub4_file1'}]}]}] self.build_tar(dir_tree) req = Request.blank('/tar_works/acc/') req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Files Created'], 4) def test_extract_tar_fail_cont_401(self): self.build_tar() req = Request.blank('/unauth/acc/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') self.assertEquals(self.app.calls, 1) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '401 Unauthorized') self.assertEquals(resp_data['Errors'], []) def test_extract_tar_fail_obj_401(self): self.build_tar() req = Request.blank('/create_obj_unauth/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '401 Unauthorized') self.assertEquals( resp_data['Errors'], [['cont/base_fails1/sub_dir1/sub1_file1', '401 Unauthorized']]) def test_extract_tar_fail_obj_name_len(self): self.build_tar() req = Request.blank('/tar_works/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') self.assertEquals(self.app.calls, 6) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Files Created'], 4) self.assertEquals( resp_data['Errors'], [['cont/base_fails1/' + ('f' * 101), '400 Bad Request']]) def test_extract_tar_fail_compress_type(self): self.build_tar() req = Request.blank('/tar_works/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, 'gz') self.assertEquals(self.app.calls, 0) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals( resp_data['Response Body'].lower(), 'invalid tar file: not a gzip file') def test_extract_tar_fail_max_failed_extractions(self): self.build_tar() with patch.object(self.bulk, 'max_failed_extractions', 1): self.app.calls = 0 req = Request.blank('/tar_works/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') self.assertEquals(self.app.calls, 5) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Files Created'], 3) self.assertEquals( resp_data['Errors'], [['cont/base_fails1/' + ('f' * 101), '400 Bad Request']]) @patch.object(bulk, 'MAX_FILE_SIZE', 4) def test_extract_tar_fail_max_file_size(self): tar = self.build_tar() dir_tree = [{'test': [{'sub_dir1': ['sub1_file1']}]}] build_dir_tree(self.testdir, dir_tree) tar = tarfile.open(name=os.path.join(self.testdir, 'tar_works.tar'), mode='w') tar.add(os.path.join(self.testdir, 'test')) tar.close() self.app.calls = 0 req = Request.blank('/tar_works/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open( os.path.join(self.testdir, 'tar_works.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') resp_data = json.loads(resp_body) self.assertEquals( resp_data['Errors'], [['cont' + self.testdir + '/test/sub_dir1/sub1_file1', '413 Request Entity Too Large']]) def test_extract_tar_fail_max_cont(self): dir_tree = [{'sub_dir1': ['sub1_file1']}, {'sub_dir2': ['sub2_file1', 'sub2_file2']}, 'f' * 101, {'sub_dir3': [{'sub4_dir1': 'sub4_file1'}]}] self.build_tar(dir_tree) with patch.object(self.bulk, 'max_containers', 1): self.app.calls = 0 body = open(os.path.join(self.testdir, 'tar_fails.tar')).read() req = Request.blank('/tar_works_cont_head_fail/acc/', body=body, headers={'Accept': 'application/json'}) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') self.assertEquals(self.app.calls, 5) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals( resp_data['Response Body'], 'More than 1 containers to create from tar.') def test_extract_tar_fail_create_cont(self): dir_tree = [{'base_fails1': [ {'sub_dir1': ['sub1_file1']}, {'sub_dir2': ['sub2_file1', 'sub2_file2']}, {'./sub_dir3': [{'sub4_dir1': 'sub4_file1'}]}]}] self.build_tar(dir_tree) req = Request.blank('/create_cont_fail/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') resp_data = json.loads(resp_body) self.assertEquals(self.app.calls, 5) self.assertEquals(len(resp_data['Errors']), 5) def test_extract_tar_fail_create_cont_value_err(self): self.build_tar() req = Request.blank('/create_cont_fail/acc/cont/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' def bad_create(req, path): raise ValueError('Test') with patch.object(self.bulk, 'create_container', bad_create): resp_body = self.handle_extract_and_iter(req, '') resp_data = json.loads(resp_body) self.assertEquals(self.app.calls, 0) self.assertEquals(len(resp_data['Errors']), 5) self.assertEquals( resp_data['Errors'][0], ['cont/base_fails1/sub_dir1/sub1_file1', '400 Bad Request']) def test_extract_tar_fail_unicode(self): dir_tree = [{'sub_dir1': ['sub1_file1']}, {'sub_dir2': ['sub2\xdefile1', 'sub2_file2']}, {'sub_\xdedir3': [{'sub4_dir1': 'sub4_file1'}]}] self.build_tar(dir_tree) req = Request.blank('/tar_works/acc/', headers={'Accept': 'application/json'}) req.environ['wsgi.input'] = open(os.path.join(self.testdir, 'tar_fails.tar')) req.headers['transfer-encoding'] = 'chunked' resp_body = self.handle_extract_and_iter(req, '') resp_data = json.loads(resp_body) self.assertEquals(self.app.calls, 4) self.assertEquals(resp_data['Number Files Created'], 2) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals( resp_data['Errors'], [['sub_dir2/sub2%DEfile1', '412 Precondition Failed'], ['sub_%DEdir3/sub4_dir1/sub4_file1', '412 Precondition Failed']]) def test_get_response_body(self): txt_body = bulk.get_response_body( 'bad_formay', {'hey': 'there'}, [['json > xml', '202 Accepted']]) self.assert_('hey: there' in txt_body) xml_body = bulk.get_response_body( 'text/xml', {'hey': 'there'}, [['json > xml', '202 Accepted']]) self.assert_('>' in xml_body) class TestDelete(unittest.TestCase): def setUp(self): self.app = FakeApp() self.bulk = bulk.filter_factory({})(self.app) def tearDown(self): self.app.calls = 0 self.app.delete_paths = [] def handle_delete_and_iter(self, req, out_content_type='application/json'): resp_body = ''.join(self.bulk.handle_delete_iter( req, out_content_type=out_content_type)) return resp_body def test_bulk_delete_uses_predefined_object_errors(self): req = Request.blank('/delete_works/AUTH_Acc') objs_to_delete = [ {'name': '/c/file_a'}, {'name': '/c/file_b', 'error': {'code': HTTP_NOT_FOUND, 'message': 'not found'}}, {'name': '/c/file_c', 'error': {'code': HTTP_UNAUTHORIZED, 'message': 'unauthorized'}}, {'name': '/c/file_d'}] resp_body = ''.join(self.bulk.handle_delete_iter( req, objs_to_delete=objs_to_delete, out_content_type='application/json')) self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/file_a', '/delete_works/AUTH_Acc/c/file_d']) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Number Deleted'], 2) self.assertEquals(resp_data['Number Not Found'], 1) self.assertEquals(resp_data['Errors'], [['/c/file_c', 'unauthorized']]) def test_bulk_delete_works_with_POST_verb(self): req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n/c/f404', headers={'Accept': 'application/json'}) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f', '/delete_works/AUTH_Acc/c/f404']) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 1) self.assertEquals(resp_data['Number Not Found'], 1) def test_bulk_delete_works_with_DELETE_verb(self): req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n/c/f404', headers={'Accept': 'application/json'}) req.method = 'DELETE' resp_body = self.handle_delete_and_iter(req) self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f', '/delete_works/AUTH_Acc/c/f404']) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 1) self.assertEquals(resp_data['Number Not Found'], 1) def test_bulk_delete_bad_content_type(self): req = Request.blank('/delete_works/AUTH_Acc', headers={'Accept': 'badformat'}) req = Request.blank('/delete_works/AUTH_Acc', headers={'Accept': 'application/json', 'Content-Type': 'text/xml'}) req.method = 'POST' req.environ['wsgi.input'] = StringIO('/c/f\n/c/f404') resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '406 Not Acceptable') def test_bulk_delete_call_and_content_type(self): def fake_start_response(*args, **kwargs): self.assertEquals(args[1][0], ('Content-Type', 'application/json')) req = Request.blank('/delete_works/AUTH_Acc?bulk-delete') req.method = 'POST' req.headers['Transfer-Encoding'] = 'chunked' req.headers['Accept'] = 'application/json' req.environ['wsgi.input'] = StringIO('/c/f%20') list(self.bulk(req.environ, fake_start_response)) # iterate over resp self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f ']) self.assertEquals(self.app.calls, 1) def test_bulk_delete_get_objs(self): req = Request.blank('/delete_works/AUTH_Acc', body='1%20\r\n2\r\n') req.method = 'POST' with patch.object(self.bulk, 'max_deletes_per_request', 2): results = self.bulk.get_objs_to_delete(req) self.assertEquals(results, [{'name': '1 '}, {'name': '2'}]) with patch.object(bulk, 'MAX_PATH_LENGTH', 2): results = [] req.environ['wsgi.input'] = StringIO('1\n2\n3') results = self.bulk.get_objs_to_delete(req) self.assertEquals(results, [{'name': '1'}, {'name': '2'}, {'name': '3'}]) with patch.object(self.bulk, 'max_deletes_per_request', 9): with patch.object(bulk, 'MAX_PATH_LENGTH', 1): req_body = '\n'.join([str(i) for i in xrange(10)]) req = Request.blank('/delete_works/AUTH_Acc', body=req_body) self.assertRaises( HTTPException, self.bulk.get_objs_to_delete, req) def test_bulk_delete_works_extra_newlines_extra_quoting(self): req = Request.blank('/delete_works/AUTH_Acc', body='/c/f\n\n\n/c/f404\n\n\n/c/%2525', headers={'Accept': 'application/json'}) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/f', '/delete_works/AUTH_Acc/c/f404', '/delete_works/AUTH_Acc/c/%25']) self.assertEquals(self.app.calls, 3) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 2) self.assertEquals(resp_data['Number Not Found'], 1) def test_bulk_delete_too_many_newlines(self): req = Request.blank('/delete_works/AUTH_Acc') req.method = 'POST' data = '\n\n' * self.bulk.max_deletes_per_request req.environ['wsgi.input'] = StringIO(data) req.content_length = len(data) resp_body = self.handle_delete_and_iter(req) self.assertTrue('413 Request Entity Too Large' in resp_body) def test_bulk_delete_works_unicode(self): body = (u'/c/ obj \u2661\r\n'.encode('utf8') + 'c/ objbadutf8\r\n' + '/c/f\xdebadutf8\n') req = Request.blank('/delete_works/AUTH_Acc', body=body, headers={'Accept': 'application/json'}) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) self.assertEquals( self.app.delete_paths, ['/delete_works/AUTH_Acc/c/ obj \xe2\x99\xa1', '/delete_works/AUTH_Acc/c/ objbadutf8']) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 1) self.assertEquals(len(resp_data['Errors']), 2) self.assertEquals( resp_data['Errors'], [[urllib.quote('c/ objbadutf8'), '412 Precondition Failed'], [urllib.quote('/c/f\xdebadutf8'), '412 Precondition Failed']]) def test_bulk_delete_no_body(self): req = Request.blank('/unauth/AUTH_acc/') resp_body = self.handle_delete_and_iter(req) self.assertTrue('411 Length Required' in resp_body) def test_bulk_delete_no_files_in_body(self): req = Request.blank('/unauth/AUTH_acc/', body=' ') resp_body = self.handle_delete_and_iter(req) self.assertTrue('400 Bad Request' in resp_body) def test_bulk_delete_unauth(self): req = Request.blank('/unauth/AUTH_acc/', body='/c/f\n/c/f_ok\n', headers={'Accept': 'application/json'}) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Errors'], [['/c/f', '401 Unauthorized']]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Number Deleted'], 1) def test_bulk_delete_500_resp(self): req = Request.blank('/broke/AUTH_acc/', body='/c/f\nc/f2\n', headers={'Accept': 'application/json'}) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals( resp_data['Errors'], [['/c/f', '500 Internal Error'], ['c/f2', '500 Internal Error']]) self.assertEquals(resp_data['Response Status'], '502 Bad Gateway') def test_bulk_delete_bad_path(self): req = Request.blank('/delete_cont_fail/') resp_body = self.handle_delete_and_iter(req) self.assertTrue('404 Not Found' in resp_body) def test_bulk_delete_container_delete(self): req = Request.blank('/delete_cont_fail/AUTH_Acc', body='c\n', headers={'Accept': 'application/json'}) req.method = 'POST' with patch('swift.common.middleware.bulk.sleep', new=mock.MagicMock(wraps=sleep, return_value=None)) as mock_sleep: resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 0) self.assertEquals(resp_data['Errors'], [['c', '409 Conflict']]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals([], mock_sleep.call_args_list) def test_bulk_delete_container_delete_retry_and_fails(self): self.bulk.retry_count = 3 req = Request.blank('/delete_cont_fail/AUTH_Acc', body='c\n', headers={'Accept': 'application/json'}) req.method = 'POST' with patch('swift.common.middleware.bulk.sleep', new=mock.MagicMock(wraps=sleep, return_value=None)) as mock_sleep: resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 0) self.assertEquals(resp_data['Errors'], [['c', '409 Conflict']]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals([call(self.bulk.retry_interval), call(self.bulk.retry_interval ** 2), call(self.bulk.retry_interval ** 3)], mock_sleep.call_args_list) def test_bulk_delete_container_delete_retry_and_success(self): self.bulk.retry_count = 3 self.app.del_container_total = 2 req = Request.blank('/delete_cont_success_after_attempts/AUTH_Acc', body='c\n', headers={'Accept': 'application/json'}) req.method = 'DELETE' with patch('swift.common.middleware.bulk.sleep', new=mock.MagicMock(wraps=sleep, return_value=None)) as mock_sleep: resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 1) self.assertEquals(resp_data['Errors'], []) self.assertEquals(resp_data['Response Status'], '200 OK') self.assertEquals([call(self.bulk.retry_interval), call(self.bulk.retry_interval ** 2)], mock_sleep.call_args_list) def test_bulk_delete_bad_file_too_long(self): req = Request.blank('/delete_works/AUTH_Acc', headers={'Accept': 'application/json'}) req.method = 'POST' bad_file = 'c/' + ('1' * bulk.MAX_PATH_LENGTH) data = '/c/f\n' + bad_file + '\n/c/f' req.environ['wsgi.input'] = StringIO(data) req.headers['Transfer-Encoding'] = 'chunked' resp_body = self.handle_delete_and_iter(req) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Number Deleted'], 2) self.assertEquals(resp_data['Errors'], [[bad_file, '400 Bad Request']]) self.assertEquals(resp_data['Response Status'], '400 Bad Request') def test_bulk_delete_bad_file_over_twice_max_length(self): body = '/c/f\nc/' + ('123456' * bulk.MAX_PATH_LENGTH) + '\n' req = Request.blank('/delete_works/AUTH_Acc', body=body) req.method = 'POST' resp_body = self.handle_delete_and_iter(req) self.assertTrue('400 Bad Request' in resp_body) def test_bulk_delete_max_failures(self): req = Request.blank('/unauth/AUTH_Acc', body='/c/f1\n/c/f2\n/c/f3', headers={'Accept': 'application/json'}) req.method = 'POST' with patch.object(self.bulk, 'max_failed_deletes', 2): resp_body = self.handle_delete_and_iter(req) self.assertEquals(self.app.calls, 2) resp_data = json.loads(resp_body) self.assertEquals(resp_data['Response Status'], '400 Bad Request') self.assertEquals(resp_data['Response Body'], 'Max delete failures exceeded') self.assertEquals(resp_data['Errors'], [['/c/f1', '401 Unauthorized'], ['/c/f2', '401 Unauthorized']]) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} utils._swift_admin_info = {} def test_registered_defaults(self): bulk.filter_factory({}) swift_info = utils.get_swift_info() self.assertTrue('bulk_upload' in swift_info) self.assertTrue(isinstance( swift_info['bulk_upload'].get('max_containers_per_extraction'), numbers.Integral)) self.assertTrue(isinstance( swift_info['bulk_upload'].get('max_failed_extractions'), numbers.Integral)) self.assertTrue('bulk_delete' in swift_info) self.assertTrue(isinstance( swift_info['bulk_delete'].get('max_deletes_per_request'), numbers.Integral)) self.assertTrue(isinstance( swift_info['bulk_delete'].get('max_failed_deletes'), numbers.Integral)) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_crossdomain.py0000664000175400017540000000572312323703611025371 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.swob import Request from swift.common.middleware import crossdomain class FakeApp(object): def __call__(self, env, start_response): return "FAKE APP" def start_response(*args): pass class TestCrossDomain(unittest.TestCase): def setUp(self): self.app = crossdomain.CrossDomainMiddleware(FakeApp(), {}) # GET of /crossdomain.xml (default) def test_crossdomain_default(self): expectedResponse = '\n' \ '\n' \ '\n' \ '\n' \ '' req = Request.blank('/crossdomain.xml', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, [expectedResponse]) # GET of /crossdomain.xml (custom) def test_crossdomain_custom(self): conf = {'cross_domain_policy': '\n'} self.app = crossdomain.CrossDomainMiddleware(FakeApp(), conf) expectedResponse = '\n' \ '\n' \ '\n' \ '\n' \ '\n' \ '' req = Request.blank('/crossdomain.xml', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, [expectedResponse]) # GET to a different resource should be passed on def test_crossdomain_pass(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') # Only GET is allowed on the /crossdomain.xml resource def test_crossdomain_get_only(self): for method in ['HEAD', 'PUT', 'POST', 'COPY', 'OPTIONS']: req = Request.blank('/crossdomain.xml', environ={'REQUEST_METHOD': method}) resp = self.app(req.environ, start_response) self.assertEquals(resp, 'FAKE APP') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_tempurl.py0000664000175400017540000011171512323703611024537 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import hmac import unittest from hashlib import sha1 from time import time from swift.common.middleware import tempauth, tempurl from swift.common.swob import Request, Response, HeaderKeyDict from swift.common import utils class FakeApp(object): def __init__(self, status_headers_body_iter=None): self.calls = 0 self.status_headers_body_iter = status_headers_body_iter if not self.status_headers_body_iter: self.status_headers_body_iter = iter([('404 Not Found', { 'x-test-header-one-a': 'value1', 'x-test-header-two-a': 'value2', 'x-test-header-two-b': 'value3'}, '')]) self.request = None def __call__(self, env, start_response): self.calls += 1 self.request = Request.blank('', environ=env) if 'swift.authorize' in env: resp = env['swift.authorize'](self.request) if resp: return resp(env, start_response) status, headers, body = self.status_headers_body_iter.next() return Response(status=status, headers=headers, body=body)(env, start_response) class TestTempURL(unittest.TestCase): def setUp(self): self.app = FakeApp() self.auth = tempauth.filter_factory({})(self.app) self.auth.reseller_prefix = 'a' self.tempurl = tempurl.filter_factory({})(self.auth) def _make_request(self, path, environ=None, keys=(), **kwargs): if environ is None: environ = {} _junk, account, _junk, _junk = utils.split_path(path, 2, 4) self._fake_cache_environ(environ, account, keys) req = Request.blank(path, environ=environ, **kwargs) return req def _fake_cache_environ(self, environ, account, keys): """ Fake out the caching layer for get_account_info(). Injects account data into environ such that keys are the tempurl keys, if set. """ meta = {'swash': 'buckle'} for idx, key in enumerate(keys): meta_name = 'Temp-URL-key' + (("-%d" % (idx + 1) if idx else "")) if key: meta[meta_name] = key environ['swift.account/' + account] = { 'status': 204, 'container_count': '0', 'total_object_count': '0', 'bytes': '0', 'meta': meta} def test_passthrough(self): resp = self._make_request('/v1/a/c/o').get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' not in resp.body) def test_allow_options(self): self.app.status_headers_body_iter = iter([('200 Ok', {}, '')]) resp = self._make_request( '/v1/a/c/o?temp_url_sig=abcde&temp_url_expires=12345', environ={'REQUEST_METHOD': 'OPTIONS'}).get_response(self.tempurl) self.assertEquals(resp.status_int, 200) def assert_valid_sig(self, expires, path, keys, sig): req = self._make_request( path, keys=keys, environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'attachment; filename="o"') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_get_valid(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() self.assert_valid_sig(expires, path, [key], sig) def test_get_valid_key2(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key1 = 'abc123' key2 = 'def456' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig1 = hmac.new(key1, hmac_body, sha1).hexdigest() sig2 = hmac.new(key2, hmac_body, sha1).hexdigest() for sig in (sig1, sig2): self.assert_valid_sig(expires, path, [key1, key2], sig) def test_get_valid_with_filename(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'filename=bob%%20%%22killer%%22.txt' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'attachment; filename="bob \\\"killer\\\".txt"') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_head_valid(self): method = 'HEAD' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'REQUEST_METHOD': 'HEAD', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) def test_get_valid_with_filename_and_inline(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'filename=bob%%20%%22killer%%22.txt&inline=' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'inline') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_get_valid_with_inline(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'inline=' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'inline') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_obj_trailing_slash(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o/' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'attachment; filename="o"') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_filename_trailing_slash(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request(path, keys=[key], environ={ 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'filename=/i/want/this/just/as/it/is/' % (sig, expires)}) self.tempurl.app = FakeApp(iter([('200 Ok', (), '123')])) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-disposition'], 'attachment; filename="/i/want/this/just/as/it/is/"') self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_get_valid_but_404(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertFalse('content-disposition' in resp.headers) self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_put_not_allowed_by_get(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'PUT', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_put_valid(self): method = 'PUT' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'PUT', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_get_not_allowed_by_put(self): method = 'PUT' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_missing_sig(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_expires=%s' % expires}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_missing_expires(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s' % sig}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_bad_path(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_no_key(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_head_allowed_by_get(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'HEAD', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_head_allowed_by_put(self): method = 'PUT' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'HEAD', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertEquals(req.environ['swift.authorize_override'], True) self.assertEquals(req.environ['REMOTE_USER'], '.wsgi.tempurl') def test_head_otherwise_not_allowed(self): method = 'PUT' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() # Deliberately fudge expires to show HEADs aren't just automatically # allowed. expires += 1 req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'HEAD', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Www-Authenticate' in resp.headers) def test_post_not_allowed(self): method = 'POST' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'POST', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_delete_not_allowed(self): method = 'DELETE' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'DELETE', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_delete_allowed_with_conf(self): self.tempurl.methods.append('DELETE') method = 'DELETE' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'DELETE', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) def test_unknown_not_allowed(self): method = 'UNKNOWN' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'REQUEST_METHOD': 'UNKNOWN', 'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_changed_path_invalid(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path + '2', keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_changed_sig_invalid(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() if sig[-1] != '0': sig = sig[:-1] + '0' else: sig = sig[:-1] + '1' req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_changed_expires_invalid(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires + 1)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_different_key_invalid(self): method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key + '2'], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) def test_removed_incoming_header(self): self.tempurl = tempurl.filter_factory({ 'incoming_remove_headers': 'x-remove-this'})(self.auth) method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], headers={'x-remove-this': 'value'}, environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertTrue('x-remove-this' not in self.app.request.headers) def test_removed_incoming_headers_match(self): self.tempurl = tempurl.filter_factory({ 'incoming_remove_headers': 'x-remove-this-*', 'incoming_allow_headers': 'x-remove-this-except-this'})(self.auth) method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], headers={'x-remove-this-one': 'value1', 'x-remove-this-except-this': 'value2'}, environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertTrue('x-remove-this-one' not in self.app.request.headers) self.assertEquals( self.app.request.headers['x-remove-this-except-this'], 'value2') def test_removed_outgoing_header(self): self.tempurl = tempurl.filter_factory({ 'outgoing_remove_headers': 'x-test-header-one-a'})(self.auth) method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertTrue('x-test-header-one-a' not in resp.headers) self.assertEquals(resp.headers['x-test-header-two-a'], 'value2') def test_removed_outgoing_headers_match(self): self.tempurl = tempurl.filter_factory({ 'outgoing_remove_headers': 'x-test-header-two-*', 'outgoing_allow_headers': 'x-test-header-two-b'})(self.auth) method = 'GET' expires = int(time() + 86400) path = '/v1/a/c/o' key = 'abc' hmac_body = '%s\n%s\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() req = self._make_request( path, keys=[key], environ={'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( sig, expires)}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 404) self.assertEquals(resp.headers['x-test-header-one-a'], 'value1') self.assertTrue('x-test-header-two-a' not in resp.headers) self.assertEquals(resp.headers['x-test-header-two-b'], 'value3') def test_get_account(self): self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'HEAD', 'PATH_INFO': '/v1/a/c/o'}), 'a') self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/o'}), 'a') self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/v1/a/c/o'}), 'a') self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'POST', 'PATH_INFO': '/v1/a/c/o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'DELETE', 'PATH_INFO': '/v1/a/c/o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'UNKNOWN', 'PATH_INFO': '/v1/a/c/o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c//////'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c///o///'}), 'a') self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a//o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1//c/o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '//a/c/o'}), None) self.assertEquals(self.tempurl._get_account({ 'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v2/a/c/o'}), None) def test_get_temp_url_info(self): s = 'f5d5051bddf5df7e27c628818738334f' e = int(time() + 86400) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( s, e)}), (s, e, None, None)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'filename=bobisyouruncle' % (s, e)}), (s, e, 'bobisyouruncle', None)) self.assertEquals( self.tempurl._get_temp_url_info({}), (None, None, None, None)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_expires=%s' % e}), (None, e, None, None)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s' % s}), (s, None, None, None)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=bad' % ( s)}), (s, 0, None, None)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'inline=' % (s, e)}), (s, e, None, True)) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s&' 'filename=bobisyouruncle&inline=' % (s, e)}), (s, e, 'bobisyouruncle', True)) e = int(time() - 1) self.assertEquals( self.tempurl._get_temp_url_info( {'QUERY_STRING': 'temp_url_sig=%s&temp_url_expires=%s' % ( s, e)}), (s, 0, None, None)) def test_get_hmacs(self): self.assertEquals( self.tempurl._get_hmacs( {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a/c/o'}, 1, ['abc']), ['026d7f7cc25256450423c7ad03fc9f5ffc1dab6d']) self.assertEquals( self.tempurl._get_hmacs( {'REQUEST_METHOD': 'HEAD', 'PATH_INFO': '/v1/a/c/o'}, 1, ['abc'], request_method='GET'), ['026d7f7cc25256450423c7ad03fc9f5ffc1dab6d']) def test_invalid(self): def _start_response(status, headers, exc_info=None): self.assertTrue(status, '401 Unauthorized') self.assertTrue('Temp URL invalid' in ''.join( self.tempurl._invalid({'REQUEST_METHOD': 'GET'}, _start_response))) self.assertEquals('', ''.join( self.tempurl._invalid({'REQUEST_METHOD': 'HEAD'}, _start_response))) def test_auth_scheme_value(self): # Passthrough environ = {} resp = self._make_request('/v1/a/c/o', environ=environ).get_response( self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' not in resp.body) self.assertTrue('Www-Authenticate' in resp.headers) self.assertTrue('swift.auth_scheme' not in environ) # Rejected by TempURL req = self._make_request('/v1/a/c/o', keys=['abc'], environ={'REQUEST_METHOD': 'PUT', 'QUERY_STRING': 'temp_url_sig=dummy&temp_url_expires=1234'}) resp = req.get_response(self.tempurl) self.assertEquals(resp.status_int, 401) self.assertTrue('Temp URL invalid' in resp.body) self.assert_('Www-Authenticate' in resp.headers) def test_clean_incoming_headers(self): irh = '' iah = '' env = {'HTTP_TEST_HEADER': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, 'incoming_allow_headers': iah} )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER' in env) irh = 'test-header' iah = '' env = {'HTTP_TEST_HEADER': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, 'incoming_allow_headers': iah} )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER' not in env) irh = 'test-header-*' iah = '' env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, 'incoming_allow_headers': iah} )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER_ONE' not in env) self.assertTrue('HTTP_TEST_HEADER_TWO' not in env) irh = 'test-header-*' iah = 'test-header-two' env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, 'incoming_allow_headers': iah} )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER_ONE' not in env) self.assertTrue('HTTP_TEST_HEADER_TWO' in env) irh = 'test-header-* test-other-header' iah = 'test-header-two test-header-yes-*' env = {'HTTP_TEST_HEADER_ONE': 'value', 'HTTP_TEST_HEADER_TWO': 'value', 'HTTP_TEST_OTHER_HEADER': 'value', 'HTTP_TEST_HEADER_YES': 'value', 'HTTP_TEST_HEADER_YES_THIS': 'value'} tempurl.TempURL( None, {'incoming_remove_headers': irh, 'incoming_allow_headers': iah} )._clean_incoming_headers(env) self.assertTrue('HTTP_TEST_HEADER_ONE' not in env) self.assertTrue('HTTP_TEST_HEADER_TWO' in env) self.assertTrue('HTTP_TEST_OTHER_HEADER' not in env) self.assertTrue('HTTP_TEST_HEADER_YES' not in env) self.assertTrue('HTTP_TEST_HEADER_YES_THIS' in env) def test_clean_outgoing_headers(self): orh = '' oah = '' hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header' in hdrs) orh = 'test-header' oah = '' hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header' not in hdrs) orh = 'test-header-*' oah = '' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' not in hdrs) orh = 'test-header-*' oah = 'test-header-two' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' in hdrs) orh = 'test-header-* test-other-header' oah = 'test-header-two test-header-yes-*' hdrs = {'test-header-one': 'value', 'test-header-two': 'value', 'test-other-header': 'value', 'test-header-yes': 'value', 'test-header-yes-this': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' in hdrs) self.assertTrue('test-other-header' not in hdrs) self.assertTrue('test-header-yes' not in hdrs) self.assertTrue('test-header-yes-this' in hdrs) def test_unicode_metadata_value(self): meta = {"temp-url-key": "test", "temp-url-key-2": u"test2"} results = tempurl.get_tempurl_keys_from_metadata(meta) for str_value in results: self.assertTrue(isinstance(str_value, str)) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} utils._swift_admin_info = {} def test_registered_defaults(self): tempurl.filter_factory({}) swift_info = utils.get_swift_info() self.assertTrue('tempurl' in swift_info) self.assertEqual(set(swift_info['tempurl']['methods']), set(('GET', 'HEAD', 'PUT'))) def test_non_default_methods(self): tempurl.filter_factory({'methods': 'GET HEAD PUT POST DELETE'}) swift_info = utils.get_swift_info() self.assertTrue('tempurl' in swift_info) self.assertEqual(set(swift_info['tempurl']['methods']), set(('GET', 'HEAD', 'PUT', 'POST', 'DELETE'))) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_formpost.py0000664000175400017540000020070412323703611024715 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import hmac import unittest from hashlib import sha1 from StringIO import StringIO from time import time from swift.common.swob import Request, Response from swift.common.middleware import tempauth, formpost from swift.common.utils import split_path class FakeApp(object): def __init__(self, status_headers_body_iter=None, check_no_query_string=True): self.status_headers_body_iter = status_headers_body_iter if not self.status_headers_body_iter: self.status_headers_body_iter = iter([('404 Not Found', { 'x-test-header-one-a': 'value1', 'x-test-header-two-a': 'value2', 'x-test-header-two-b': 'value3'}, '')]) self.requests = [] self.check_no_query_string = check_no_query_string def __call__(self, env, start_response): if self.check_no_query_string and env.get('QUERY_STRING'): raise Exception('Query string %s should have been discarded!' % env['QUERY_STRING']) body = '' while True: chunk = env['wsgi.input'].read() if not chunk: break body += chunk env['wsgi.input'] = StringIO(body) self.requests.append(Request.blank('', environ=env)) if env.get('swift.authorize_override') and \ env.get('REMOTE_USER') != '.wsgi.pre_authed': raise Exception( 'Invalid REMOTE_USER %r with swift.authorize_override' % ( env.get('REMOTE_USER'),)) if 'swift.authorize' in env: resp = env['swift.authorize'](self.requests[-1]) if resp: return resp(env, start_response) status, headers, body = self.status_headers_body_iter.next() return Response(status=status, headers=headers, body=body)(env, start_response) class TestParseAttrs(unittest.TestCase): def test_basic_content_type(self): name, attrs = formpost._parse_attrs('text/plain') self.assertEquals(name, 'text/plain') self.assertEquals(attrs, {}) def test_content_type_with_charset(self): name, attrs = formpost._parse_attrs('text/plain; charset=UTF8') self.assertEquals(name, 'text/plain') self.assertEquals(attrs, {'charset': 'UTF8'}) def test_content_disposition(self): name, attrs = formpost._parse_attrs( 'form-data; name="somefile"; filename="test.html"') self.assertEquals(name, 'form-data') self.assertEquals(attrs, {'name': 'somefile', 'filename': 'test.html'}) class TestIterRequests(unittest.TestCase): def test_bad_start(self): it = formpost._iter_requests(StringIO('blah'), 'unique') exc = None try: it.next() except formpost.FormInvalid as err: exc = err self.assertEquals(str(exc), 'invalid starting boundary') def test_empty(self): it = formpost._iter_requests(StringIO('--unique'), 'unique') fp = it.next() self.assertEquals(fp.read(), '') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_basic(self): it = formpost._iter_requests( StringIO('--unique\r\nabcdefg\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.read(), 'abcdefg') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_basic2(self): it = formpost._iter_requests( StringIO('--unique\r\nabcdefg\r\n--unique\r\nhijkl\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.read(), 'abcdefg') fp = it.next() self.assertEquals(fp.read(), 'hijkl') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_tiny_reads(self): it = formpost._iter_requests( StringIO('--unique\r\nabcdefg\r\n--unique\r\nhijkl\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.read(2), 'ab') self.assertEquals(fp.read(2), 'cd') self.assertEquals(fp.read(2), 'ef') self.assertEquals(fp.read(2), 'g') self.assertEquals(fp.read(2), '') fp = it.next() self.assertEquals(fp.read(), 'hijkl') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_big_reads(self): it = formpost._iter_requests( StringIO('--unique\r\nabcdefg\r\n--unique\r\nhijkl\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.read(65536), 'abcdefg') self.assertEquals(fp.read(), '') fp = it.next() self.assertEquals(fp.read(), 'hijkl') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_broken_mid_stream(self): # We go ahead and accept whatever is sent instead of rejecting the # whole request, in case the partial form is still useful. it = formpost._iter_requests( StringIO('--unique\r\nabc'), 'unique') fp = it.next() self.assertEquals(fp.read(), 'abc') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_readline(self): it = formpost._iter_requests( StringIO('--unique\r\nab\r\ncd\ref\ng\r\n--unique\r\nhi\r\n\r\n' 'jkl\r\n\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.readline(), 'ab\r\n') self.assertEquals(fp.readline(), 'cd\ref\ng') self.assertEquals(fp.readline(), '') fp = it.next() self.assertEquals(fp.readline(), 'hi\r\n') self.assertEquals(fp.readline(), '\r\n') self.assertEquals(fp.readline(), 'jkl\r\n') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) def test_readline_with_tiny_chunks(self): orig_read_chunk_size = formpost.READ_CHUNK_SIZE try: formpost.READ_CHUNK_SIZE = 2 it = formpost._iter_requests( StringIO('--unique\r\nab\r\ncd\ref\ng\r\n--unique\r\nhi\r\n' '\r\njkl\r\n\r\n--unique--'), 'unique') fp = it.next() self.assertEquals(fp.readline(), 'ab\r\n') self.assertEquals(fp.readline(), 'cd\ref\ng') self.assertEquals(fp.readline(), '') fp = it.next() self.assertEquals(fp.readline(), 'hi\r\n') self.assertEquals(fp.readline(), '\r\n') self.assertEquals(fp.readline(), 'jkl\r\n') exc = None try: it.next() except StopIteration as err: exc = err self.assertTrue(exc is not None) finally: formpost.READ_CHUNK_SIZE = orig_read_chunk_size class TestCappedFileLikeObject(unittest.TestCase): def test_whole(self): self.assertEquals( formpost._CappedFileLikeObject(StringIO('abc'), 10).read(), 'abc') def test_exceeded(self): exc = None try: formpost._CappedFileLikeObject(StringIO('abc'), 2).read() except EOFError as err: exc = err self.assertEquals(str(exc), 'max_file_size exceeded') def test_whole_readline(self): fp = formpost._CappedFileLikeObject(StringIO('abc\ndef'), 10) self.assertEquals(fp.readline(), 'abc\n') self.assertEquals(fp.readline(), 'def') self.assertEquals(fp.readline(), '') def test_exceeded_readline(self): fp = formpost._CappedFileLikeObject(StringIO('abc\ndef'), 5) self.assertEquals(fp.readline(), 'abc\n') exc = None try: self.assertEquals(fp.readline(), 'def') except EOFError as err: exc = err self.assertEquals(str(exc), 'max_file_size exceeded') def test_read_sized(self): fp = formpost._CappedFileLikeObject(StringIO('abcdefg'), 10) self.assertEquals(fp.read(2), 'ab') self.assertEquals(fp.read(2), 'cd') self.assertEquals(fp.read(2), 'ef') self.assertEquals(fp.read(2), 'g') self.assertEquals(fp.read(2), '') class TestFormPost(unittest.TestCase): def setUp(self): self.app = FakeApp() self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) def _make_request(self, path, tempurl_keys=(), **kwargs): req = Request.blank(path, **kwargs) # Fake out the caching layer so that get_account_info() finds its # data. Include something that isn't tempurl keys to prove we skip it. meta = {'user-job-title': 'Personal Trainer', 'user-real-name': 'Jim Shortz'} for idx, key in enumerate(tempurl_keys): meta_name = 'temp-url-key' + (("-%d" % (idx + 1) if idx else "")) if key: meta[meta_name] = key _junk, account, _junk, _junk = split_path(path, 2, 4) req.environ['swift.account/' + account] = self._fake_cache_env( account, tempurl_keys) return req def _fake_cache_env(self, account, tempurl_keys=()): # Fake out the caching layer so that get_account_info() finds its # data. Include something that isn't tempurl keys to prove we skip it. meta = {'user-job-title': 'Personal Trainer', 'user-real-name': 'Jim Shortz'} for idx, key in enumerate(tempurl_keys): meta_name = 'temp-url-key' + ("-%d" % (idx + 1) if idx else "") if key: meta[meta_name] = key return {'status': 204, 'container_count': '0', 'total_object_count': '0', 'bytes': '0', 'meta': meta} def _make_sig_env_body(self, path, redirect, max_file_size, max_file_count, expires, key, user_agent=True): sig = hmac.new( key, '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() body = [ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', redirect, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="expires"', '', str(expires), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="signature"', '', sig, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file1"; ' 'filename="testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file2"; ' 'filename="testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', ] wsgi_errors = StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate', 'HTTP_ACCEPT_LANGUAGE': 'en-us', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;' 'q=0.9,*/*;q=0.8', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_HOST': 'ubuntu:8080', 'HTTP_ORIGIN': 'file://', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X ' '10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) ' 'Version/5.1.2 Safari/534.52.7', 'PATH_INFO': path, 'REMOTE_ADDR': '172.16.83.1', 'REQUEST_METHOD': 'POST', 'SCRIPT_NAME': '', 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'wsgi.errors': wsgi_errors, 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), } if user_agent is False: del env['HTTP_USER_AGENT'] return sig, env, body def test_passthrough(self): for method in ('HEAD', 'GET', 'PUT', 'POST', 'DELETE'): resp = self._make_request( '/v1/a/c/o', environ={'REQUEST_METHOD': method}).get_response(self.formpost) self.assertEquals(resp.status_int, 401) self.assertTrue('FormPost' not in resp.body) def test_auth_scheme(self): # FormPost rejects key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') authenticate_v = None for h, v in headers: if h.lower() == 'www-authenticate': authenticate_v = v self.assertTrue('FormPost: Form Expired' in body) self.assertEquals('Swift realm="unknown"', authenticate_v) def test_safari(self): key = 'abc' path = '/v1/AUTH_test/container' redirect = 'http://brim.net' max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig = hmac.new( key, '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() wsgi_input = StringIO('\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', redirect, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="expires"', '', str(expires), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="signature"', '', sig, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file1"; ' 'filename="testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file2"; ' 'filename="testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', ])) wsgi_errors = StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate', 'HTTP_ACCEPT_LANGUAGE': 'en-us', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;' 'q=0.9,*/*;q=0.8', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_HOST': 'ubuntu:8080', 'HTTP_ORIGIN': 'file://', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X ' '10_7_2) AppleWebKit/534.52.7 (KHTML, like Gecko) ' 'Version/5.1.2 Safari/534.52.7', 'PATH_INFO': path, 'REMOTE_ADDR': '172.16.83.1', 'REQUEST_METHOD': 'POST', 'SCRIPT_NAME': '', 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), } self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://brim.net?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue('http://brim.net?status=201&message=' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_firefox(self): key = 'abc' path = '/v1/AUTH_test/container' redirect = 'http://brim.net' max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig = hmac.new( key, '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() wsgi_input = StringIO('\r\n'.join([ '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="redirect"', '', redirect, '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="expires"', '', str(expires), '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="signature"', '', sig, '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="file1"; ' 'filename="testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="file2"; ' 'filename="testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '-----------------------------168072824752491622650073', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '-----------------------------168072824752491622650073--', '' ])) wsgi_errors = StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=---------------------------168072824752491622650073', 'HTTP_ACCEPT_CHARSET': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate', 'HTTP_ACCEPT_LANGUAGE': 'en-us,en;q=0.5', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;' 'q=0.9,*/*;q=0.8', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_HOST': 'ubuntu:8080', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; ' 'rv:8.0.1) Gecko/20100101 Firefox/8.0.1', 'PATH_INFO': '/v1/AUTH_test/container', 'REMOTE_ADDR': '172.16.83.1', 'REQUEST_METHOD': 'POST', 'SCRIPT_NAME': '', 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), } self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://brim.net?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue('http://brim.net?status=201&message=' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_chrome(self): key = 'abc' path = '/v1/AUTH_test/container' redirect = 'http://brim.net' max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig = hmac.new( key, '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() wsgi_input = StringIO('\r\n'.join([ '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="redirect"', '', redirect, '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="expires"', '', str(expires), '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="signature"', '', sig, '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="file1"; ' 'filename="testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="file2"; ' 'filename="testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '------WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '------WebKitFormBoundaryq3CFxUjfsDMu8XsA--', '' ])) wsgi_errors = StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=----WebKitFormBoundaryq3CFxUjfsDMu8XsA', 'HTTP_ACCEPT_CHARSET': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'HTTP_ACCEPT_ENCODING': 'gzip,deflate,sdch', 'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8', 'HTTP_ACCEPT': 'text/html,application/xhtml+xml,application/xml;' 'q=0.9,*/*;q=0.8', 'HTTP_CACHE_CONTROL': 'max-age=0', 'HTTP_CONNECTION': 'keep-alive', 'HTTP_HOST': 'ubuntu:8080', 'HTTP_ORIGIN': 'null', 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X ' '10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) ' 'Chrome/16.0.912.63 Safari/535.7', 'PATH_INFO': '/v1/AUTH_test/container', 'REMOTE_ADDR': '172.16.83.1', 'REQUEST_METHOD': 'POST', 'SCRIPT_NAME': '', 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), } self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://brim.net?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue('http://brim.net?status=201&message=' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_explorer(self): key = 'abc' path = '/v1/AUTH_test/container' redirect = 'http://brim.net' max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig = hmac.new( key, '%s\n%s\n%s\n%s\n%s' % ( path, redirect, max_file_size, max_file_count, expires), sha1).hexdigest() wsgi_input = StringIO('\r\n'.join([ '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="redirect"', '', redirect, '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="expires"', '', str(expires), '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="signature"', '', sig, '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="file1"; ' 'filename="C:\\testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="file2"; ' 'filename="C:\\testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '-----------------------------7db20d93017c', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '-----------------------------7db20d93017c--', '' ])) wsgi_errors = StringIO() env = { 'CONTENT_TYPE': 'multipart/form-data; ' 'boundary=---------------------------7db20d93017c', 'HTTP_ACCEPT_ENCODING': 'gzip, deflate', 'HTTP_ACCEPT_LANGUAGE': 'en-US', 'HTTP_ACCEPT': 'text/html, application/xhtml+xml, */*', 'HTTP_CACHE_CONTROL': 'no-cache', 'HTTP_CONNECTION': 'Keep-Alive', 'HTTP_HOST': '172.16.83.128:8080', 'HTTP_USER_AGENT': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT ' '6.1; WOW64; Trident/5.0)', 'PATH_INFO': '/v1/AUTH_test/container', 'REMOTE_ADDR': '172.16.83.129', 'REQUEST_METHOD': 'POST', 'SCRIPT_NAME': '', 'SERVER_NAME': '172.16.83.128', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'swift.account/AUTH_test': self._fake_cache_env( 'AUTH_test', [key]), 'wsgi.errors': wsgi_errors, 'wsgi.input': wsgi_input, 'wsgi.multiprocess': False, 'wsgi.multithread': True, 'wsgi.run_once': False, 'wsgi.url_scheme': 'http', 'wsgi.version': (1, 0), } self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://brim.net?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue('http://brim.net?status=201&message=' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_messed_up_start(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('XX' + '\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) def log_assert_int_status(env, response_status_int): self.assertTrue(isinstance(response_status_int, int)) self.formpost._log_request = log_assert_int_status status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '400 Bad Request') self.assertEquals(exc_info, None) self.assertTrue('FormPost: invalid starting boundary' in body) self.assertEquals(len(self.app.requests), 0) def test_max_file_size_exceeded(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 5, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '400 Bad Request') self.assertEquals(exc_info, None) self.assertTrue('FormPost: max_file_size exceeded' in body) self.assertEquals(len(self.app.requests), 0) def test_max_file_count_exceeded(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 1024, 1, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals( location, 'http://brim.net?status=400&message=max%20file%20count%20exceeded') self.assertEquals(exc_info, None) self.assertTrue( 'http://brim.net?status=400&message=max%20file%20count%20exceeded' in body) self.assertEquals(len(self.app.requests), 1) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') def test_subrequest_does_not_pass_query(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['QUERY_STRING'] = 'this=should¬=get&passed' env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp( iter([('201 Created', {}, ''), ('201 Created', {}, '')]), check_no_query_string=True) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] # Make sure we 201 Created, which means we made the final subrequest # (and FakeApp verifies that no QUERY_STRING got passed). self.assertEquals(status, '201 Created') self.assertEquals(exc_info, None) self.assertTrue('201 Created' in body) self.assertEquals(len(self.app.requests), 2) def test_subrequest_fails(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://brim.net', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('404 Not Found', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://brim.net?status=404&message=') self.assertEquals(exc_info, None) self.assertTrue('http://brim.net?status=404&message=' in body) self.assertEquals(len(self.app.requests), 1) def test_truncated_attr_value(self): key = 'abc' redirect = 'a' * formpost.MAX_VALUE_LENGTH max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', redirect, max_file_size, max_file_count, expires, key) # Tack on an extra char to redirect, but shouldn't matter since it # should get truncated off on read. redirect += 'b' env['wsgi.input'] = StringIO('\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', redirect, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="expires"', '', str(expires), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="signature"', '', sig, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file1"; ' 'filename="testfile1.txt"', 'Content-Type: text/plain', '', 'Test File\nOne\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file2"; ' 'filename="testfile2.txt"', 'Content-Type: text/plain', '', 'Test\nFile\nTwo\n', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="file3"; filename=""', 'Content-Type: application/octet-stream', '', '', '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', ])) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals( location, ('a' * formpost.MAX_VALUE_LENGTH) + '?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue( ('a' * formpost.MAX_VALUE_LENGTH) + '?status=201&message=' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_no_file_to_process(self): key = 'abc' redirect = 'http://brim.net' max_file_size = 1024 max_file_count = 10 expires = int(time() + 86400) sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', redirect, max_file_size, max_file_count, expires, key) env['wsgi.input'] = StringIO('\r\n'.join([ '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="redirect"', '', redirect, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_size"', '', str(max_file_size), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="max_file_count"', '', str(max_file_count), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="expires"', '', str(expires), '------WebKitFormBoundaryNcxTqxSlX7t4TDkR', 'Content-Disposition: form-data; name="signature"', '', sig, '------WebKitFormBoundaryNcxTqxSlX7t4TDkR--', '', ])) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals( location, 'http://brim.net?status=400&message=no%20files%20to%20process') self.assertEquals(exc_info, None) self.assertTrue( 'http://brim.net?status=400&message=no%20files%20to%20process' in body) self.assertEquals(len(self.app.requests), 0) def test_formpost_without_useragent(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) def start_response(s, h, e=None): pass body = ''.join(self.formpost(env, start_response)) self.assertTrue('User-Agent' in self.app.requests[0].headers) self.assertEquals(self.app.requests[0].headers['User-Agent'], 'FormPost') def test_formpost_with_origin(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key, user_agent=False) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) env['HTTP_ORIGIN'] = 'http://localhost:5000' self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {'Access-Control-Allow-Origin': 'http://localhost:5000'}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) headers = {} def start_response(s, h, e=None): for k, v in h: headers[k] = v pass body = ''.join(self.formpost(env, start_response)) self.assertEquals(headers['Access-Control-Allow-Origin'], 'http://localhost:5000') def test_formpost_with_multiple_keys(self): key = 'ernie' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) # Stick it in X-Account-Meta-Temp-URL-Key-2 and make sure we get it env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', ['bert', key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h body = ''.join(self.formpost(env, start_response)) self.assertEqual('303 See Other', status[0]) self.assertEqual( 'http://redirect?status=201&message=', dict(headers[0]).get('Location')) def test_redirect(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://redirect?status=201&message=') self.assertEquals(exc_info, None) self.assertTrue(location in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_redirect_with_query(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', 'http://redirect?one=two', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '303 See Other') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, 'http://redirect?one=two&status=201&message=') self.assertEquals(exc_info, None) self.assertTrue(location in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_no_redirect(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '201 Created') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('201 Created' in body) self.assertEquals(len(self.app.requests), 2) self.assertEquals(self.app.requests[0].body, 'Test File\nOne\n') self.assertEquals(self.app.requests[1].body, 'Test\nFile\nTwo\n') def test_no_redirect_expired(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() - 10), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Form Expired' in body) def test_no_redirect_invalid_sig(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) # Change key to invalidate sig env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key + ' is bogus now']) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_no_redirect_with_error(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('XX' + '\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '400 Bad Request') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: invalid starting boundary' in body) def test_no_v1(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v2/AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_empty_v1(self): key = 'abc' sig, env, body = self._make_sig_env_body( '//AUTH_test/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_empty_account(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1//container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_wrong_account(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_tst/container', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([ ('200 Ok', {'x-account-meta-temp-url-key': 'def'}, ''), ('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_no_container(self): key = 'abc' sig, env, body = self._make_sig_env_body( '/v1/AUTH_test', '', 1024, 10, int(time() + 86400), key) env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '401 Unauthorized') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: Invalid Signature' in body) def test_completely_non_int_expires(self): key = 'abc' expires = int(time() + 86400) sig, env, body = self._make_sig_env_body( '/v1/AUTH_test/container', '', 1024, 10, expires, key) for i, v in enumerate(body): if v == str(expires): body[i] = 'badvalue' break env['wsgi.input'] = StringIO('\r\n'.join(body)) env['swift.account/AUTH_test'] = self._fake_cache_env( 'AUTH_test', [key]) self.app = FakeApp(iter([('201 Created', {}, ''), ('201 Created', {}, '')])) self.auth = tempauth.filter_factory({})(self.app) self.formpost = formpost.filter_factory({})(self.auth) status = [None] headers = [None] exc_info = [None] def start_response(s, h, e=None): status[0] = s headers[0] = h exc_info[0] = e body = ''.join(self.formpost(env, start_response)) status = status[0] headers = headers[0] exc_info = exc_info[0] self.assertEquals(status, '400 Bad Request') location = None for h, v in headers: if h.lower() == 'location': location = v self.assertEquals(location, None) self.assertEquals(exc_info, None) self.assertTrue('FormPost: expired not an integer' in body) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_dlo.py0000664000175400017540000012314112323703614023624 0ustar jenkinsjenkins00000000000000#-*- coding:utf-8 -*- # Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import contextlib import hashlib import json import mock import tempfile import time import unittest from swift.common import exceptions, swob from swift.common.middleware import dlo from test.unit.common.middleware.helpers import FakeSwift from textwrap import dedent LIMIT = 'swift.common.middleware.dlo.CONTAINER_LISTING_LIMIT' def md5hex(s): return hashlib.md5(s).hexdigest() class DloTestCase(unittest.TestCase): def call_dlo(self, req, app=None, expect_exception=False): if app is None: app = self.dlo req.headers.setdefault("User-Agent", "Soap Opera") status = [None] headers = [None] def start_response(s, h, ei=None): status[0] = s headers[0] = h body_iter = app(req.environ, start_response) body = '' caught_exc = None try: for chunk in body_iter: body += chunk except Exception as exc: if expect_exception: caught_exc = exc else: raise if expect_exception: return status[0], headers[0], body, caught_exc else: return status[0], headers[0], body def setUp(self): self.app = FakeSwift() self.dlo = dlo.filter_factory({ # don't slow down tests with rate limiting 'rate_limit_after_segment': '1000000', })(self.app) self.app.register( 'GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("aaaaa")}, 'aaaaa') self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, 'bbbbb') self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ccccc")}, 'ccccc') self.app.register( 'GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("ddddd")}, 'ddddd') self.app.register( 'GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("eeeee")}, 'eeeee') # an unrelated object (not seg*) to test the prefix matching self.app.register( 'GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk, {'Content-Length': '9', 'Etag': md5hex("meow meow meow meow")}, 'meow meow meow meow') self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg'}, 'manifest-contents') lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{"hash": md5hex("aaaaa"), "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct}, {"hash": md5hex("bbbbb"), "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct}, {"hash": md5hex("ccccc"), "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct}, {"hash": md5hex("ddddd"), "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct}, {"hash": md5hex("eeeee"), "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct}] full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9, "name": "catpicture.jpg", "last_modified": lm, "content_type": "application/png"}] self.app.register( 'GET', '/v1/AUTH_test/c?format=json', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(full_container_listing)) self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs)) # This is to let us test multi-page container listings; we use the # trailing underscore to send small (pagesize=3) listings. # # If you're testing against this, be sure to mock out # CONTAINER_LISTING_LIMIT to 3 in your test. self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg', 'X-Object-Manifest': 'c/seg_'}, 'manyseg') self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[:3])) self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[3:])) # Here's a manifest with 0 segments self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg', 'X-Object-Manifest': 'c/noseg_'}, 'noseg') self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps([])) class TestDloPutManifest(DloTestCase): def setUp(self): super(TestDloPutManifest, self).setUp() self.app.register( 'PUT', '/v1/AUTH_test/c/m', swob.HTTPCreated, {}, None) def test_validating_x_object_manifest(self): exp_okay = ["c/o", "c/obj/with/slashes", "c/obj/with/trailing/slash/", "c/obj/with//multiple///slashes////adjacent"] exp_bad = ["", "/leading/slash", "double//slash", "container-only", "whole-container/", "c/o?short=querystring", "c/o?has=a&long-query=string"] got_okay = [] got_bad = [] for val in (exp_okay + exp_bad): req = swob.Request.blank("/v1/AUTH_test/c/m", environ={'REQUEST_METHOD': 'PUT'}, headers={"X-Object-Manifest": val}) status, _, _ = self.call_dlo(req) if status.startswith("201"): got_okay.append(val) else: got_bad.append(val) self.assertEqual(exp_okay, got_okay) self.assertEqual(exp_bad, got_bad) def test_validation_watches_manifests_with_slashes(self): self.app.register( 'PUT', '/v1/AUTH_test/con/w/x/y/z', swob.HTTPCreated, {}, None) req = swob.Request.blank( "/v1/AUTH_test/con/w/x/y/z", environ={'REQUEST_METHOD': 'PUT'}, headers={"X-Object-Manifest": 'good/value'}) status, _, _ = self.call_dlo(req) self.assertEqual(status, "201 Created") req = swob.Request.blank( "/v1/AUTH_test/con/w/x/y/z", environ={'REQUEST_METHOD': 'PUT'}, headers={"X-Object-Manifest": '/badvalue'}) status, _, _ = self.call_dlo(req) self.assertEqual(status, "400 Bad Request") def test_validation_ignores_containers(self): self.app.register( 'PUT', '/v1/a/c', swob.HTTPAccepted, {}, None) req = swob.Request.blank( "/v1/a/c", environ={'REQUEST_METHOD': 'PUT'}, headers={"X-Object-Manifest": "/superbogus/?wrong=in&every=way"}) status, _, _ = self.call_dlo(req) self.assertEqual(status, "202 Accepted") def test_validation_ignores_accounts(self): self.app.register( 'PUT', '/v1/a', swob.HTTPAccepted, {}, None) req = swob.Request.blank( "/v1/a", environ={'REQUEST_METHOD': 'PUT'}, headers={"X-Object-Manifest": "/superbogus/?wrong=in&every=way"}) status, _, _ = self.call_dlo(req) self.assertEqual(status, "202 Accepted") class TestDloHeadManifest(DloTestCase): def test_head_large_object(self): expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Content-Length"], "25") def test_head_large_object_too_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'HEAD'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) # etag is manifest's etag self.assertEqual(headers["Etag"], "etag-manyseg") self.assertEqual(headers.get("Content-Length"), None) def test_head_large_object_no_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-no-segments', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(headers["Etag"], '"%s"' % md5hex("")) self.assertEqual(headers["Content-Length"], "0") # one request to HEAD the manifest # one request for the first page of listings # *zero* requests for the second page of listings self.assertEqual( self.app.calls, [('HEAD', '/v1/AUTH_test/mancon/manifest-no-segments'), ('GET', '/v1/AUTH_test/c?format=json&prefix=noseg_')]) class TestDloGetManifest(DloTestCase): def test_get_manifest(self): expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(headers["Etag"], expected_etag) self.assertEqual(headers["Content-Length"], "25") self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') for _, _, hdrs in self.app.calls_with_headers[1:]: ua = hdrs.get("User-Agent", "") self.assertTrue("DLO MultipartGET" in ua) self.assertFalse("DLO MultipartGET DLO MultipartGET" in ua) # the first request goes through unaltered self.assertFalse( "DLO MultipartGET" in self.app.calls_with_headers[0][2]) # we set swift.source for everything but the first request self.assertEqual(self.app.swift_sources, [None, 'DLO', 'DLO', 'DLO', 'DLO', 'DLO', 'DLO']) def test_get_non_manifest_passthrough(self): req = swob.Request.blank('/v1/AUTH_test/c/catpicture.jpg', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(body, "meow meow meow meow") def test_get_non_object_passthrough(self): self.app.register('GET', '/info', swob.HTTPOk, {}, 'useful stuff here') req = swob.Request.blank('/info', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, '200 OK') self.assertEqual(body, 'useful stuff here') self.assertEqual(self.app.call_count, 1) def test_get_manifest_passthrough(self): # reregister it with the query param self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest?multipart-manifest=get', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg'}, 'manifest-contents') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'multipart-manifest=get'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(headers["Etag"], "manifest-etag") self.assertEqual(body, "manifest-contents") def test_error_passthrough(self): self.app.register( 'GET', '/v1/AUTH_test/gone/404ed', swob.HTTPNotFound, {}, None) req = swob.Request.blank('/v1/AUTH_test/gone/404ed', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, '404 Not Found') def test_get_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=8-17'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") self.assertEqual(body, "bbcccccddd") expected_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) self.assertEqual(headers.get("Etag"), expected_etag) def test_get_range_on_segment_boundaries(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=10-19'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") self.assertEqual(body, "cccccddddd") def test_get_range_first_byte(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-0'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "1") self.assertEqual(body, "a") def test_get_range_last_byte(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=24-24'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "1") self.assertEqual(body, "e") def test_get_range_overlapping_end(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=18-30'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "7") self.assertEqual(headers["Content-Range"], "bytes 18-24/25") self.assertEqual(body, "ddeeeee") def test_get_range_unsatisfiable(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=25-30'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, "416 Requested Range Not Satisfiable") def test_get_range_many_segments_satisfiable(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=3-12'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "10") # The /15 here indicates that this is a 15-byte object. DLO can't tell # if there are more segments or not without fetching more container # listings, though, so we just go with the sum of the lengths of the # segments we can see. In an ideal world, this would be "bytes 3-12/*" # to indicate that we don't know the full object length. However, RFC # 2616 section 14.16 explicitly forbids us from doing that: # # A response with status code 206 (Partial Content) MUST NOT include # a Content-Range field with a byte-range-resp-spec of "*". # # Since the truth is forbidden, we lie. self.assertEqual(headers["Content-Range"], "bytes 3-12/15") self.assertEqual(body, "aabbbbbccc") self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/mancon/manifest-many-segments'), ('GET', '/v1/AUTH_test/c?format=json&prefix=seg_'), ('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')]) def test_get_range_many_segments_satisfiability_unknown(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=10-22'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "200 OK") # this requires multiple pages of container listing, so we can't send # a Content-Length header self.assertEqual(headers.get("Content-Length"), None) self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") def test_get_suffix_range(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-40'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "206 Partial Content") self.assertEqual(headers["Content-Length"], "25") self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") def test_get_suffix_range_many_segments(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-5'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(headers.get("Content-Length"), None) self.assertEqual(headers.get("Content-Range"), None) self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") def test_get_multi_range(self): # DLO doesn't support multi-range GETs. The way that you express that # in HTTP is to return a 200 response containing the whole entity. req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=5-9,15-19'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, "200 OK") self.assertEqual(headers.get("Content-Length"), None) self.assertEqual(headers.get("Content-Range"), None) self.assertEqual(body, "aaaaabbbbbcccccdddddeeeee") def test_if_match_matches(self): manifest_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': manifest_etag}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') def test_if_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': 'not it'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '412 Precondition Failed') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(body, '') def test_if_none_match_matches(self): manifest_etag = '"%s"' % md5hex( md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") + md5hex("ddddd") + md5hex("eeeee")) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': manifest_etag}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '304 Not Modified') self.assertEqual(headers['Content-Length'], '0') self.assertEqual(body, '') def test_if_none_match_does_not_match(self): req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': 'not it'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') self.assertEqual(headers['Content-Length'], '25') self.assertEqual(body, 'aaaaabbbbbcccccdddddeeeee') def test_get_with_if_modified_since(self): # It's important not to pass the If-[Un]Modified-Since header to the # proxy for segment GET requests, as it may result in 304 Not Modified # responses, and those don't contain segment data. req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': 'Wed, 12 Feb 2014 22:24:52 GMT', 'If-Unmodified-Since': 'Thu, 13 Feb 2014 23:25:53 GMT'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) for _, _, hdrs in self.app.calls_with_headers[1:]: self.assertFalse('If-Modified-Since' in hdrs) self.assertFalse('If-Unmodified-Since' in hdrs) def test_error_fetching_first_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_01', swob.HTTPForbidden, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual(status, "200 OK") self.assertEqual(body, '') # error right away -> no body bytes sent def test_error_fetching_second_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPForbidden, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaa") # first segment made it out def test_error_listing_container_first_listing_request(self): self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_', swob.HTTPNotFound, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-5'}) with mock.patch(LIMIT, 3): status, headers, body = self.call_dlo(req) self.assertEqual(status, "404 Not Found") def test_error_listing_container_second_listing_request(self): self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03', swob.HTTPNotFound, {}, None) req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=-5'}) with mock.patch(LIMIT, 3): status, headers, body, exc = self.call_dlo( req, expect_exception=True) self.assertTrue(isinstance(exc, exceptions.ListingIterError)) self.assertEqual(status, "200 OK") self.assertEqual(body, "aaaaabbbbbccccc") def test_mismatched_etag_fetching_second_segment(self): self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("bbbbb")}, 'bbWRONGbb') req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual(status, "200 OK") self.assertEqual(''.join(body), "aaaaabbWRONGbb") # stop after error def test_etag_comparison_ignores_quotes(self): # a little future-proofing here in case we ever fix this self.app.register( 'HEAD', '/v1/AUTH_test/mani/festo', swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah', 'X-Object-Manifest': 'c/quotetags'}, None) self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=quotetags', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps([{"hash": "\"abc\"", "bytes": 5, "name": "quotetags1", "last_modified": "2013-11-22T02:42:14.261620", "content-type": "application/octet-stream"}, {"hash": "def", "bytes": 5, "name": "quotetags2", "last_modified": "2013-11-22T02:42:14.261620", "content-type": "application/octet-stream"}])) req = swob.Request.blank('/v1/AUTH_test/mani/festo', environ={'REQUEST_METHOD': 'HEAD'}) status, headers, body = self.call_dlo(req) headers = swob.HeaderKeyDict(headers) self.assertEqual(headers["Etag"], '"' + hashlib.md5("abcdef").hexdigest() + '"') def test_object_prefix_quoting(self): self.app.register( 'GET', '/v1/AUTH_test/man/accent', swob.HTTPOk, {'Content-Length': '0', 'Etag': 'blah', 'X-Object-Manifest': u'c/é'.encode('utf-8')}, None) segs = [{"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é1"}, {"hash": md5hex("AAAAA"), "bytes": 5, "name": u"é2"}] self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=%C3%A9', swob.HTTPOk, {'Content-Type': 'application/json'}, json.dumps(segs)) self.app.register( 'GET', '/v1/AUTH_test/c/\xC3\xa91', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("AAAAA")}, "AAAAA") self.app.register( 'GET', '/v1/AUTH_test/c/\xC3\xA92', swob.HTTPOk, {'Content-Length': '5', 'Etag': md5hex("BBBBB")}, "BBBBB") req = swob.Request.blank('/v1/AUTH_test/man/accent', environ={'REQUEST_METHOD': 'GET'}) status, headers, body = self.call_dlo(req) self.assertEqual(status, "200 OK") self.assertEqual(body, "AAAAABBBBB") def test_get_taking_too_long(self): the_time = [time.time()] def mock_time(): return the_time[0] # this is just a convenient place to hang a time jump def mock_is_success(status_int): the_time[0] += 9 * 3600 return status_int // 100 == 2 req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) with contextlib.nested( mock.patch('swift.common.request_helpers.time.time', mock_time), mock.patch('swift.common.request_helpers.is_success', mock_is_success), mock.patch.object(dlo, 'is_success', mock_is_success)): status, headers, body, exc = self.call_dlo( req, expect_exception=True) self.assertEqual(status, '200 OK') self.assertEqual(body, 'aaaaabbbbbccccc') self.assertTrue(isinstance(exc, exceptions.SegmentError)) def test_get_oversize_segment(self): # If we send a Content-Length header to the client, it's based on the # container listing. If a segment gets bigger by the time we get to it # (like if a client uploads a bigger segment w/the same name), we need # to not send anything beyond the length we promised. Also, we should # probably raise an exception. # This is now longer than the original seg_03+seg_04+seg_05 combined self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'}, 'cccccccccccccccccccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(body, 'aaaaabbbbbccccccccccccccc') self.assertTrue(isinstance(exc, exceptions.SegmentError)) self.assertEqual( self.app.calls, [('GET', '/v1/AUTH_test/mancon/manifest'), ('GET', '/v1/AUTH_test/c?format=json&prefix=seg'), ('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'), ('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')]) def test_get_undersize_segment(self): # If we send a Content-Length header to the client, it's based on the # container listing. If a segment gets smaller by the time we get to # it (like if a client uploads a smaller segment w/the same name), we # need to raise an exception so that the connection will be closed by # the WSGI server. Otherwise, the WSGI server will be waiting for the # next request, the client will still be waiting for the rest of the # response, and nobody will be happy. # Shrink it by a single byte self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")}, 'cccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '200 OK') # sanity check self.assertEqual(headers.get('Content-Length'), '25') # sanity check self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee') self.assertTrue(isinstance(exc, exceptions.SegmentError)) def test_get_undersize_segment_range(self): # Shrink it by a single byte self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")}, 'cccc') req = swob.Request.blank( '/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET'}, headers={'Range': 'bytes=0-14'}) status, headers, body, exc = self.call_dlo(req, expect_exception=True) headers = swob.HeaderKeyDict(headers) self.assertEqual(status, '206 Partial Content') # sanity check self.assertEqual(headers.get('Content-Length'), '15') # sanity check self.assertEqual(body, 'aaaaabbbbbcccc') self.assertTrue(isinstance(exc, exceptions.SegmentError)) def test_get_with_auth_overridden(self): auth_got_called = [0] def my_auth(): auth_got_called[0] += 1 return None req = swob.Request.blank('/v1/AUTH_test/mancon/manifest', environ={'REQUEST_METHOD': 'GET', 'swift.authorize': my_auth}) status, headers, body = self.call_dlo(req) self.assertTrue(auth_got_called[0] > 1) def fake_start_response(*args, **kwargs): pass class TestDloCopyHook(DloTestCase): def setUp(self): super(TestDloCopyHook, self).setUp() self.app.register( 'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, {'Content-Length': '10', 'Etag': 'o1-etag'}, "aaaaaaaaaa") self.app.register( 'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, {'Content-Length': '10', 'Etag': 'o2-etag'}, "bbbbbbbbbb") self.app.register( 'GET', '/v1/AUTH_test/c/man', swob.HTTPOk, {'X-Object-Manifest': 'c/o'}, "manifest-contents") lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1", "last_modified": lm, "content_type": ct}, {"hash": "o2-etag", "bytes": 5, "name": "o2", "last_modified": lm, "content_type": ct}] self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=o', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs)) copy_hook = [None] # slip this guy in there to pull out the hook def extract_copy_hook(env, sr): copy_hook[0] = env.get('swift.copy_hook') return self.app(env, sr) self.dlo = dlo.filter_factory({})(extract_copy_hook) req = swob.Request.blank('/v1/AUTH_test/c/o1', environ={'REQUEST_METHOD': 'GET'}) self.dlo(req.environ, fake_start_response) self.copy_hook = copy_hook[0] self.assertTrue(self.copy_hook is not None) # sanity check def test_copy_hook_passthrough(self): source_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'GET'}) sink_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'PUT'}) source_resp = swob.Response(request=source_req, status=200) # no X-Object-Manifest header, so do nothing modified_resp = self.copy_hook(source_req, source_resp, sink_req) self.assertTrue(modified_resp is source_resp) def test_copy_hook_manifest(self): source_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'GET'}) sink_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'PUT'}) source_resp = swob.Response( request=source_req, status=200, headers={"X-Object-Manifest": "c/o"}, app_iter=["manifest"]) # it's a manifest, so copy the segments to make a normal object modified_resp = self.copy_hook(source_req, source_resp, sink_req) self.assertTrue(modified_resp is not source_resp) self.assertEqual(modified_resp.etag, hashlib.md5("o1-etago2-etag").hexdigest()) self.assertEqual(sink_req.headers.get('X-Object-Manifest'), None) def test_copy_hook_manifest_with_multipart_manifest_get(self): source_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'GET', 'QUERY_STRING': 'multipart-manifest=get'}) sink_req = swob.Request.blank( '/v1/AUTH_test/c/man', environ={'REQUEST_METHOD': 'PUT'}) source_resp = swob.Response( request=source_req, status=200, headers={"X-Object-Manifest": "c/o"}, app_iter=["manifest"]) # make sure the sink request (the backend PUT) gets X-Object-Manifest # on it, but that's all modified_resp = self.copy_hook(source_req, source_resp, sink_req) self.assertTrue(modified_resp is source_resp) self.assertEqual(sink_req.headers.get('X-Object-Manifest'), 'c/o') class TestDloConfiguration(unittest.TestCase): """ For backwards compatibility, we will read a couple of values out of the proxy's config section if we don't have any config values. """ def test_skip_defaults_if_configured(self): # The presence of even one config value in our config section means we # won't go looking for the proxy config at all. proxy_conf = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server [filter:dlo] use = egg:swift#dlo max_get_time = 3600 [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_segments_per_sec = 7 rate_limit_after_segment = 13 max_get_time = 2900 """) conffile = tempfile.NamedTemporaryFile() conffile.write(proxy_conf) conffile.flush() mware = dlo.filter_factory({ 'max_get_time': '3600', '__file__': conffile.name })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(10, mware.rate_limit_after_segment) self.assertEqual(3600, mware.max_get_time) def test_finding_defaults_from_file(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 max_get_time = 2900 """) conffile = tempfile.NamedTemporaryFile() conffile.write(proxy_conf) conffile.flush() mware = dlo.filter_factory({ '__file__': conffile.name })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) def test_finding_defaults_from_dir(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf1 = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server """) proxy_conf2 = dedent(""" [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 max_get_time = 2900 """) conf_dir = tempfile.mkdtemp() conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile1.write(proxy_conf1) conffile1.flush() conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile2.write(proxy_conf2) conffile2.flush() mware = dlo.filter_factory({ '__file__': conf_dir })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/test_name_check.py0000664000175400017540000000651412323703611025124 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. ''' Unit tests for Name_check filter Created on February 29, 2012 @author: eamonn-otoole ''' import unittest from swift.common.swob import Request, Response from swift.common.middleware import name_check MAX_LENGTH = 255 FORBIDDEN_CHARS = '\'\"<>`' FORBIDDEN_REGEXP = "/\./|/\.\./|/\.$|/\.\.$" class FakeApp(object): def __call__(self, env, start_response): return Response(body="OK")(env, start_response) class TestNameCheckMiddleware(unittest.TestCase): def setUp(self): self.conf = {'maximum_length': MAX_LENGTH, 'forbidden_chars': FORBIDDEN_CHARS, 'forbidden_regexp': FORBIDDEN_REGEXP} self.test_check = name_check.filter_factory(self.conf)(FakeApp()) def test_valid_length_and_character(self): path = '/V1.0/' + 'c' * (MAX_LENGTH - 6) resp = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'} ).get_response(self.test_check) self.assertEquals(resp.body, 'OK') def test_invalid_character(self): for c in self.conf['forbidden_chars']: path = '/V1.0/1234' + c + '5' resp = Request.blank( path, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.test_check) self.assertEquals( resp.body, ("Object/Container name contains forbidden chars from %s" % self.conf['forbidden_chars'])) self.assertEquals(resp.status_int, 400) def test_invalid_length(self): path = '/V1.0/' + 'c' * (MAX_LENGTH - 5) resp = Request.blank(path, environ={'REQUEST_METHOD': 'PUT'} ).get_response(self.test_check) self.assertEquals( resp.body, ("Object/Container name longer than the allowed maximum %s" % self.conf['maximum_length'])) self.assertEquals(resp.status_int, 400) def test_invalid_regexp(self): for s in ['/.', '/..', '/./foo', '/../foo']: path = '/V1.0/' + s resp = Request.blank( path, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.test_check) self.assertEquals( resp.body, ("Object/Container name contains a forbidden substring " "from regular expression %s" % self.conf['forbidden_regexp'])) self.assertEquals(resp.status_int, 400) def test_valid_regexp(self): for s in ['/...', '/.\.', '/foo']: path = '/V1.0/' + s resp = Request.blank( path, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.test_check) self.assertEquals(resp.body, 'OK') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/middleware/__init__.py0000664000175400017540000000000012323703611023527 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/test_direct_client.py0000664000175400017540000002714012323703611023540 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import os import StringIO from hashlib import md5 from swift.common import direct_client from swift.common.exceptions import ClientException from swift.common.utils import json def mock_http_connect(status, fake_headers=None, body=None): class FakeConn(object): def __init__(self, status, fake_headers, body, *args, **kwargs): self.status = status self.reason = 'Fake' self.body = body self.host = args[0] self.port = args[1] self.method = args[4] self.path = args[5] self.with_exc = False self.headers = kwargs.get('headers', {}) self.fake_headers = fake_headers self.etag = md5() def getresponse(self): if self.with_exc: raise Exception('test') if self.fake_headers is not None and self.method == 'POST': self.fake_headers.append(self.headers) return self def getheader(self, header, default=None): return self.headers.get(header.lower(), default) def getheaders(self): if self.fake_headers is not None: for key in self.fake_headers: self.headers.update({key: self.fake_headers[key]}) return self.headers.items() def read(self): return self.body def send(self, data): self.etag.update(data) self.headers['etag'] = str(self.etag.hexdigest()) def close(self): return return lambda *args, **kwargs: FakeConn(status, fake_headers, body, *args, **kwargs) class TestDirectClient(unittest.TestCase): def test_gen_headers(self): hdrs = direct_client.gen_headers() assert 'user-agent' in hdrs assert hdrs['user-agent'] == 'direct-client %s' % os.getpid() assert len(hdrs.keys()) == 1 hdrs = direct_client.gen_headers(add_ts=True) assert 'user-agent' in hdrs assert 'x-timestamp' in hdrs assert len(hdrs.keys()) == 2 hdrs = direct_client.gen_headers(hdrs_in={'foo-bar': '47'}) assert 'user-agent' in hdrs assert 'foo-bar' in hdrs assert hdrs['foo-bar'] == '47' assert len(hdrs.keys()) == 2 hdrs = direct_client.gen_headers(hdrs_in={'user-agent': '47'}) assert 'user-agent' in hdrs assert hdrs['user-agent'] == 'direct-client %s' % os.getpid() assert len(hdrs.keys()) == 1 def test_direct_get_account(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' headers = { 'X-Account-Container-Count': '1', 'X-Account-Object-Count': '1', 'X-Account-Bytes-Used': '1', 'X-Timestamp': '1234567890', 'X-PUT-Timestamp': '1234567890'} body = '[{"count": 1, "bytes": 20971520, "name": "c1"}]' fake_headers = {} for header, value in headers.items(): fake_headers[header.lower()] = value was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, fake_headers, body) resp_headers, resp = direct_client.direct_get_account(node, part, account) fake_headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(fake_headers, resp_headers) self.assertEqual(json.loads(body), resp) direct_client.http_connect = mock_http_connect(204, fake_headers, body) resp_headers, resp = direct_client.direct_get_account(node, part, account) fake_headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(fake_headers, resp_headers) self.assertEqual([], resp) direct_client.http_connect = was_http_connector def test_direct_head_container(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' headers = {'key': 'value'} was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers) resp = direct_client.direct_head_container(node, part, account, container) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp) direct_client.http_connect = was_http_connector def test_direct_get_container(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' headers = {'key': 'value'} body = '[{"hash": "8f4e3", "last_modified": "317260", "bytes": 209}]' was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers, body) resp_headers, resp = ( direct_client.direct_get_container(node, part, account, container)) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp_headers) self.assertEqual(json.loads(body), resp) direct_client.http_connect = mock_http_connect(204, headers, body) resp_headers, resp = ( direct_client.direct_get_container(node, part, account, container)) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp_headers) self.assertEqual([], resp) direct_client.http_connect = was_http_connector def test_direct_delete_container(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200) direct_client.direct_delete_container(node, part, account, container) direct_client.http_connect = was_http_connector def test_direct_head_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' headers = {'key': 'value'} was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers) resp = direct_client.direct_head_object(node, part, account, container, name) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp) direct_client.http_connect = was_http_connector def test_direct_get_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' contents = StringIO.StringIO('123456') was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, body=contents) resp_header, obj_body = ( direct_client.direct_get_object(node, part, account, container, name)) self.assertEqual(obj_body, contents) direct_client.http_connect = was_http_connector pass def test_direct_post_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' headers = {'Key': 'value'} fake_headers = [] was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, fake_headers) direct_client.direct_post_object(node, part, account, container, name, headers) self.assertEqual(headers['Key'], fake_headers[0].get('Key')) direct_client.http_connect = was_http_connector def test_direct_delete_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200) direct_client.direct_delete_object(node, part, account, container, name) direct_client.http_connect = was_http_connector def test_direct_put_object(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' contents = StringIO.StringIO('123456') was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200) resp = direct_client.direct_put_object(node, part, account, container, name, contents, 6) self.assertEqual(md5('123456').hexdigest(), resp) direct_client.http_connect = was_http_connector def test_direct_put_object_fail(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' contents = StringIO.StringIO('123456') was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(500) self.assertRaises(ClientException, direct_client.direct_put_object, node, part, account, container, name, contents) direct_client.http_connect = was_http_connector def test_direct_put_object_chunked(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' contents = StringIO.StringIO('123456') was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200) resp = direct_client.direct_put_object(node, part, account, container, name, contents) self.assertEqual(md5('6\r\n123456\r\n0\r\n\r\n').hexdigest(), resp) direct_client.http_connect = was_http_connector def test_retry(self): node = {'ip': '1.2.3.4', 'port': '6000', 'device': 'sda'} part = '0' account = 'a' container = 'c' name = 'o' headers = {'key': 'value'} was_http_connector = direct_client.http_connect direct_client.http_connect = mock_http_connect(200, headers) attempts, resp = direct_client.retry(direct_client.direct_head_object, node, part, account, container, name) headers.update({'user-agent': 'direct-client %s' % os.getpid()}) self.assertEqual(headers, resp) self.assertEqual(attempts, 1) direct_client.http_connect = was_http_connector if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/corrupted_example.db0000664000175400017540000002000012323703611023334 0ustar jenkinsjenkins00000000000000junkte format 3@      F''Ktableoutgoing_syncoutgoing_syncCREATE TABLE outgoing_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 )9M'indexsqlite_autoindex_outgoing_sync_1outgoing_syncF''Ktableincoming_syncincoming_syncCREATE TABLE incoming_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 )9M'indexsqlite_autoindex_incoming_sync_1incoming_sync5']triggeroutgoing_sync_insertoutgoing_syncCREATE TRIGGER outgoing_sync_insert AFTER INSERT ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END }}0 EtabletesttestCREATE TABLE test (one TEXT)5']triggerincoming_sync_updateincoming_syncCREATE TRIGGER incoming_sync_update AFTER UPDATE ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END5']triggerincoming_sync_insertincoming_syncCREATE TRIGGER incoming_sync_insert AFTER INSERT ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END5']triggeroutgoing_sync_updateoutgoing_syncCREATE TRIGGER outgoing_sync_update AFTER UPDATE ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END 1swift-1.13.1/test/unit/common/test_wsgi.py0000664000175400017540000011423112323703611021677 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.common.wsgi""" import errno import logging import mimetools import socket import unittest import os import pickle from textwrap import dedent from gzip import GzipFile from contextlib import nested from StringIO import StringIO from collections import defaultdict from contextlib import closing from urllib import quote from eventlet import listen import mock import swift.common.middleware.catch_errors import swift.common.middleware.gatekeeper import swift.proxy.server from swift.common.swob import Request from swift.common import wsgi, utils, ring from test.unit import temptree from paste.deploy import loadwsgi def _fake_rings(tmpdir): account_ring_path = os.path.join(tmpdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': 6012}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': 6022}], 30), f) container_ring_path = os.path.join(tmpdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': 6011}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': 6021}], 30), f) object_ring_path = os.path.join(tmpdir, 'object.ring.gz') with closing(GzipFile(object_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': 6010}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': 6020}], 30), f) class TestWSGI(unittest.TestCase): """Tests for swift.common.wsgi""" def setUp(self): utils.HASH_PATH_PREFIX = 'startcap' self._orig_parsetype = mimetools.Message.parsetype def tearDown(self): mimetools.Message.parsetype = self._orig_parsetype def test_monkey_patch_mimetools(self): sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).type, 'text/plain') sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).plisttext, '') sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).maintype, 'text') sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).subtype, 'plain') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).type, 'text/html') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).plisttext, '; charset=ISO-8859-4') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).maintype, 'text') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).subtype, 'html') wsgi.monkey_patch_mimetools() sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).type, None) sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).plisttext, '') sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).maintype, None) sio = StringIO('blah') self.assertEquals(mimetools.Message(sio).subtype, None) sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).type, 'text/html') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).plisttext, '; charset=ISO-8859-4') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).maintype, 'text') sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') self.assertEquals(mimetools.Message(sio).subtype, 'html') def test_init_request_processor(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app, conf, logger, log_name = wsgi.init_request_processor( conf_file, 'proxy-server') # verify pipeline is catch_errors -> dlo -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) app = app.app expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware self.assert_(isinstance(app, expected)) app = app.app expected = swift.common.middleware.dlo.DynamicLargeObject self.assert_(isinstance(app, expected)) app = app.app expected = swift.proxy.server.Application self.assert_(isinstance(app, expected)) # config settings applied to app instance self.assertEquals(0.2, app.conn_timeout) # appconfig returns values from 'proxy-server' section expected = { '__file__': conf_file, 'here': os.path.dirname(conf_file), 'conn_timeout': '0.2', 'swift_dir': t, } self.assertEquals(expected, conf) # logger works logger.info('testing') self.assertEquals('proxy-server', log_name) def test_init_request_processor_from_conf_dir(self): config_dir = { 'proxy-server.conf.d/pipeline.conf': """ [pipeline:main] pipeline = catch_errors proxy-server """, 'proxy-server.conf.d/app.conf': """ [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 """, 'proxy-server.conf.d/catch-errors.conf': """ [filter:catch_errors] use = egg:swift#catch_errors """ } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with mock.patch('swift.proxy.server.Application.modify_wsgi_pipeline'): with temptree(*zip(*config_dir.items())) as conf_root: conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: f.write('[DEFAULT]\nswift_dir = %s' % conf_root) _fake_rings(conf_root) app, conf, logger, log_name = wsgi.init_request_processor( conf_dir, 'proxy-server') # verify pipeline is catch_errors -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) self.assert_(isinstance(app.app, swift.proxy.server.Application)) # config settings applied to app instance self.assertEquals(0.2, app.app.conn_timeout) # appconfig returns values from 'proxy-server' section expected = { '__file__': conf_dir, 'here': conf_dir, 'conn_timeout': '0.2', 'swift_dir': conf_root, } self.assertEquals(expected, conf) # logger works logger.info('testing') self.assertEquals('proxy-server', log_name) def test_get_socket(self): # stubs conf = {} ssl_conf = { 'cert_file': '', 'key_file': '', } # mocks class MockSocket(object): def __init__(self): self.opts = defaultdict(dict) def setsockopt(self, level, optname, value): self.opts[level][optname] = value def mock_listen(*args, **kwargs): return MockSocket() class MockSsl(object): def __init__(self): self.wrap_socket_called = [] def wrap_socket(self, sock, **kwargs): self.wrap_socket_called.append(kwargs) return sock # patch old_listen = wsgi.listen old_ssl = wsgi.ssl try: wsgi.listen = mock_listen wsgi.ssl = MockSsl() # test sock = wsgi.get_socket(conf) # assert self.assert_(isinstance(sock, MockSocket)) expected_socket_opts = { socket.SOL_SOCKET: { socket.SO_REUSEADDR: 1, socket.SO_KEEPALIVE: 1, }, socket.IPPROTO_TCP: { socket.TCP_NODELAY: 1, } } if hasattr(socket, 'TCP_KEEPIDLE'): expected_socket_opts[socket.IPPROTO_TCP][ socket.TCP_KEEPIDLE] = 600 self.assertEquals(sock.opts, expected_socket_opts) # test ssl sock = wsgi.get_socket(ssl_conf) expected_kwargs = { 'certfile': '', 'keyfile': '', } self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs]) finally: wsgi.listen = old_listen wsgi.ssl = old_ssl def test_address_in_use(self): # stubs conf = {} # mocks def mock_listen(*args, **kwargs): raise socket.error(errno.EADDRINUSE) def value_error_listen(*args, **kwargs): raise ValueError('fake') def mock_sleep(*args): pass class MockTime(object): """Fast clock advances 10 seconds after every call to time """ def __init__(self): self.current_time = old_time.time() def time(self, *args, **kwargs): rv = self.current_time # advance for next call self.current_time += 10 return rv old_listen = wsgi.listen old_sleep = wsgi.sleep old_time = wsgi.time try: wsgi.listen = mock_listen wsgi.sleep = mock_sleep wsgi.time = MockTime() # test error self.assertRaises(Exception, wsgi.get_socket, conf) # different error wsgi.listen = value_error_listen self.assertRaises(ValueError, wsgi.get_socket, conf) finally: wsgi.listen = old_listen wsgi.sleep = old_sleep wsgi.time = old_time def test_run_server(self): config = """ [DEFAULT] eventlet_debug = yes client_timeout = 30 max_clients = 1000 swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy # while "set" values normally override default set client_timeout = 20 # this section is not in conf during run_server set max_clients = 10 """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch('swift.proxy.server.Application.' 'modify_wsgi_pipeline'): with mock.patch('swift.common.wsgi.wsgi') as _wsgi: with mock.patch('swift.common.wsgi.eventlet') as _eventlet: conf = wsgi.appconfig(conf_file) logger = logging.getLogger('test') sock = listen(('localhost', 0)) wsgi.run_server(conf, logger, sock) self.assertEquals('HTTP/1.0', _wsgi.HttpProtocol.default_request_version) self.assertEquals(30, _wsgi.WRITE_TIMEOUT) _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True) _eventlet.debug.hub_exceptions.assert_called_with(True) _wsgi.server.assert_called() args, kwargs = _wsgi.server.call_args server_sock, server_app, server_logger = args self.assertEquals(sock, server_sock) self.assert_(isinstance(server_app, swift.proxy.server.Application)) self.assertEquals(20, server_app.client_timeout) self.assert_(isinstance(server_logger, wsgi.NullLogger)) self.assert_('custom_pool' in kwargs) self.assertEquals(1000, kwargs['custom_pool'].size) def test_run_server_conf_dir(self): config_dir = { 'proxy-server.conf.d/pipeline.conf': """ [pipeline:main] pipeline = proxy-server """, 'proxy-server.conf.d/app.conf': """ [app:proxy-server] use = egg:swift#proxy """, 'proxy-server.conf.d/default.conf': """ [DEFAULT] eventlet_debug = yes client_timeout = 30 """ } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as conf_root: conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: f.write('[DEFAULT]\nswift_dir = %s' % conf_root) _fake_rings(conf_root) with mock.patch('swift.proxy.server.Application.' 'modify_wsgi_pipeline'): with mock.patch('swift.common.wsgi.wsgi') as _wsgi: with mock.patch('swift.common.wsgi.eventlet') as _eventlet: with mock.patch.dict('os.environ', {'TZ': ''}): conf = wsgi.appconfig(conf_dir) logger = logging.getLogger('test') sock = listen(('localhost', 0)) wsgi.run_server(conf, logger, sock) self.assert_(os.environ['TZ'] is not '') self.assertEquals('HTTP/1.0', _wsgi.HttpProtocol.default_request_version) self.assertEquals(30, _wsgi.WRITE_TIMEOUT) _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True) _eventlet.debug.hub_exceptions.assert_called_with(True) _wsgi.server.assert_called() args, kwargs = _wsgi.server.call_args server_sock, server_app, server_logger = args self.assertEquals(sock, server_sock) self.assert_(isinstance(server_app, swift.proxy.server.Application)) self.assert_(isinstance(server_logger, wsgi.NullLogger)) self.assert_('custom_pool' in kwargs) def test_appconfig_dir_ignores_hidden_files(self): config_dir = { 'server.conf.d/01.conf': """ [app:main] use = egg:swift#proxy port = 8080 """, 'server.conf.d/.01.conf.swp': """ [app:main] use = egg:swift#proxy port = 8081 """, } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as path: conf_dir = os.path.join(path, 'server.conf.d') conf = wsgi.appconfig(conf_dir) expected = { '__file__': os.path.join(path, 'server.conf.d'), 'here': os.path.join(path, 'server.conf.d'), 'port': '8080', } self.assertEquals(conf, expected) def test_pre_auth_wsgi_input(self): oldenv = {} newenv = wsgi.make_pre_authed_env(oldenv) self.assertTrue('wsgi.input' in newenv) self.assertEquals(newenv['wsgi.input'].read(), '') oldenv = {'wsgi.input': StringIO('original wsgi.input')} newenv = wsgi.make_pre_authed_env(oldenv) self.assertTrue('wsgi.input' in newenv) self.assertEquals(newenv['wsgi.input'].read(), '') oldenv = {'swift.source': 'UT'} newenv = wsgi.make_pre_authed_env(oldenv) self.assertEquals(newenv['swift.source'], 'UT') oldenv = {'swift.source': 'UT'} newenv = wsgi.make_pre_authed_env(oldenv, swift_source='SA') self.assertEquals(newenv['swift.source'], 'SA') def test_pre_auth_req(self): class FakeReq(object): @classmethod def fake_blank(cls, path, environ={}, body='', headers={}): self.assertEquals(environ['swift.authorize']('test'), None) self.assertFalse('HTTP_X_TRANS_ID' in environ) was_blank = Request.blank Request.blank = FakeReq.fake_blank wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'}, 'PUT', '/', body='tester', headers={}) wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'}, 'PUT', '/', headers={}) Request.blank = was_blank def test_pre_auth_req_with_quoted_path(self): r = wsgi.make_pre_authed_request( {'HTTP_X_TRANS_ID': '1234'}, 'PUT', path=quote('/a space'), body='tester', headers={}) self.assertEquals(r.path, quote('/a space')) def test_pre_auth_req_drops_query(self): r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path') self.assertEquals(r.query_string, 'original') r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path?replacement') self.assertEquals(r.query_string, 'replacement') r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path?') self.assertEquals(r.query_string, '') def test_pre_auth_req_with_body(self): r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path', 'the body') self.assertEquals(r.body, 'the body') def test_pre_auth_creates_script_name(self): e = wsgi.make_pre_authed_env({}) self.assertTrue('SCRIPT_NAME' in e) def test_pre_auth_copies_script_name(self): e = wsgi.make_pre_authed_env({'SCRIPT_NAME': '/script_name'}) self.assertEquals(e['SCRIPT_NAME'], '/script_name') def test_pre_auth_copies_script_name_unless_path_overridden(self): e = wsgi.make_pre_authed_env({'SCRIPT_NAME': '/script_name'}, path='/override') self.assertEquals(e['SCRIPT_NAME'], '') self.assertEquals(e['PATH_INFO'], '/override') def test_pre_auth_req_swift_source(self): r = wsgi.make_pre_authed_request( {'QUERY_STRING': 'original'}, 'GET', 'path', 'the body', swift_source='UT') self.assertEquals(r.body, 'the body') self.assertEquals(r.environ['swift.source'], 'UT') def test_run_server_global_conf_callback(self): calls = defaultdict(lambda: 0) def _initrp(conf_file, app_section, *args, **kwargs): return ( {'__file__': 'test', 'workers': 0}, 'logger', 'log_name') def _global_conf_callback(preloaded_app_conf, global_conf): calls['_global_conf_callback'] += 1 self.assertEqual( preloaded_app_conf, {'__file__': 'test', 'workers': 0}) self.assertEqual(global_conf, {'log_name': 'log_name'}) global_conf['test1'] = 'one' def _loadapp(uri, name=None, **kwargs): calls['_loadapp'] += 1 self.assertTrue('global_conf' in kwargs) self.assertEqual(kwargs['global_conf'], {'log_name': 'log_name', 'test1': 'one'}) with nested( mock.patch.object(wsgi, '_initrp', _initrp), mock.patch.object(wsgi, 'get_socket'), mock.patch.object(wsgi, 'drop_privileges'), mock.patch.object(wsgi, 'loadapp', _loadapp), mock.patch.object(wsgi, 'capture_stdio'), mock.patch.object(wsgi, 'run_server')): wsgi.run_wsgi('conf_file', 'app_section', global_conf_callback=_global_conf_callback) self.assertEqual(calls['_global_conf_callback'], 1) self.assertEqual(calls['_loadapp'], 1) def test_run_server_success(self): calls = defaultdict(lambda: 0) def _initrp(conf_file, app_section, *args, **kwargs): calls['_initrp'] += 1 return ( {'__file__': 'test', 'workers': 0}, 'logger', 'log_name') def _loadapp(uri, name=None, **kwargs): calls['_loadapp'] += 1 with nested( mock.patch.object(wsgi, '_initrp', _initrp), mock.patch.object(wsgi, 'get_socket'), mock.patch.object(wsgi, 'drop_privileges'), mock.patch.object(wsgi, 'loadapp', _loadapp), mock.patch.object(wsgi, 'capture_stdio'), mock.patch.object(wsgi, 'run_server')): rc = wsgi.run_wsgi('conf_file', 'app_section') self.assertEqual(calls['_initrp'], 1) self.assertEqual(calls['_loadapp'], 1) self.assertEqual(rc, 0) def test_run_server_failure1(self): calls = defaultdict(lambda: 0) def _initrp(conf_file, app_section, *args, **kwargs): calls['_initrp'] += 1 raise wsgi.ConfigFileError('test exception') def _loadapp(uri, name=None, **kwargs): calls['_loadapp'] += 1 with nested( mock.patch.object(wsgi, '_initrp', _initrp), mock.patch.object(wsgi, 'get_socket'), mock.patch.object(wsgi, 'drop_privileges'), mock.patch.object(wsgi, 'loadapp', _loadapp), mock.patch.object(wsgi, 'capture_stdio'), mock.patch.object(wsgi, 'run_server')): rc = wsgi.run_wsgi('conf_file', 'app_section') self.assertEqual(calls['_initrp'], 1) self.assertEqual(calls['_loadapp'], 0) self.assertEqual(rc, 1) def test_pre_auth_req_with_empty_env_no_path(self): r = wsgi.make_pre_authed_request( {}, 'GET') self.assertEquals(r.path, quote('')) self.assertTrue('SCRIPT_NAME' in r.environ) self.assertTrue('PATH_INFO' in r.environ) def test_pre_auth_req_with_env_path(self): r = wsgi.make_pre_authed_request( {'PATH_INFO': '/unquoted path with %20'}, 'GET') self.assertEquals(r.path, quote('/unquoted path with %20')) self.assertEquals(r.environ['SCRIPT_NAME'], '') def test_pre_auth_req_with_env_script(self): r = wsgi.make_pre_authed_request({'SCRIPT_NAME': '/hello'}, 'GET') self.assertEquals(r.path, quote('/hello')) def test_pre_auth_req_with_env_path_and_script(self): env = {'PATH_INFO': '/unquoted path with %20', 'SCRIPT_NAME': '/script'} r = wsgi.make_pre_authed_request(env, 'GET') expected_path = quote(env['SCRIPT_NAME'] + env['PATH_INFO']) self.assertEquals(r.path, expected_path) env = {'PATH_INFO': '', 'SCRIPT_NAME': '/script'} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '/script') env = {'PATH_INFO': '/path', 'SCRIPT_NAME': ''} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '/path') env = {'PATH_INFO': '', 'SCRIPT_NAME': ''} r = wsgi.make_pre_authed_request(env, 'GET') self.assertEquals(r.path, '') def test_pre_auth_req_path_overrides_env(self): env = {'PATH_INFO': '/path', 'SCRIPT_NAME': '/script'} r = wsgi.make_pre_authed_request(env, 'GET', '/override') self.assertEquals(r.path, '/override') self.assertEquals(r.environ['SCRIPT_NAME'], '') self.assertEquals(r.environ['PATH_INFO'], '/override') class TestWSGIContext(unittest.TestCase): def test_app_call(self): statuses = ['200 Ok', '404 Not Found'] def app(env, start_response): start_response(statuses.pop(0), [('Content-Length', '3')]) yield 'Ok\n' wc = wsgi.WSGIContext(app) r = Request.blank('/') it = wc._app_call(r.environ) self.assertEquals(wc._response_status, '200 Ok') self.assertEquals(''.join(it), 'Ok\n') r = Request.blank('/') it = wc._app_call(r.environ) self.assertEquals(wc._response_status, '404 Not Found') self.assertEquals(''.join(it), 'Ok\n') def test_app_iter_is_closable(self): def app(env, start_response): start_response('200 OK', [('Content-Length', '25')]) yield 'aaaaa' yield 'bbbbb' yield 'ccccc' yield 'ddddd' yield 'eeeee' wc = wsgi.WSGIContext(app) r = Request.blank('/') iterable = wc._app_call(r.environ) self.assertEquals(wc._response_status, '200 OK') iterator = iter(iterable) self.assertEqual('aaaaa', iterator.next()) self.assertEqual('bbbbb', iterator.next()) iterable.close() self.assertRaises(StopIteration, iterator.next) class TestPipelineWrapper(unittest.TestCase): def setUp(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck catch_errors tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:healthcheck] use = egg:swift#healthcheck [filter:tempurl] paste.filter_factory = swift.common.middleware.tempurl:filter_factory """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={}) self.pipe = wsgi.PipelineWrapper(ctx) def _entry_point_names(self): # Helper method to return a list of the entry point names for the # filters in the pipeline. return [c.entry_point_name for c in self.pipe.context.filter_contexts] def test_startswith(self): self.assertTrue(self.pipe.startswith("healthcheck")) self.assertFalse(self.pipe.startswith("tempurl")) def test_startswith_no_filters(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={}) pipe = wsgi.PipelineWrapper(ctx) self.assertTrue(pipe.startswith('proxy')) def test_insert_filter(self): original_modules = ['healthcheck', 'catch_errors', None] self.assertEqual(self._entry_point_names(), original_modules) self.pipe.insert_filter(self.pipe.create_filter('catch_errors')) expected_modules = ['catch_errors', 'healthcheck', 'catch_errors', None] self.assertEqual(self._entry_point_names(), expected_modules) def test_str(self): self.assertEqual( str(self.pipe), "healthcheck catch_errors " + "swift.common.middleware.tempurl:filter_factory proxy") def test_str_unknown_filter(self): self.pipe.context.filter_contexts[0].entry_point_name = None self.pipe.context.filter_contexts[0].object = 'mysterious' self.assertEqual( str(self.pipe), " catch_errors " + "swift.common.middleware.tempurl:filter_factory proxy") class TestPipelineModification(unittest.TestCase): def pipeline_modules(self, app): # This is rather brittle; it'll break if a middleware stores its app # anywhere other than an attribute named "app", but it works for now. pipe = [] for _ in xrange(1000): pipe.append(app.__class__.__module__) if not hasattr(app, 'app'): break app = app.app return pipe def test_load_app(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:healthcheck] use = egg:swift#healthcheck """ def modify_func(app, pipe): new = pipe.create_filter('catch_errors') pipe.insert_filter(new) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', modify_func): app = wsgi.loadapp(conf_file, global_conf={}) exp = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app.app, exp), app.app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app.app, exp), app.app.app) # make sure you can turn off the pipeline modification if you want def blow_up(*_, **__): raise self.fail("needs more struts") with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', blow_up): app = wsgi.loadapp(conf_file, global_conf={}, allow_modify_pipeline=False) # the pipeline was untouched exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app, exp), app.app) def test_proxy_unmodified_wsgi_pipeline(self): # Make sure things are sane even when we modify nothing config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = catch_errors gatekeeper proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', 'swift.common.middleware.healthcheck', 'swift.proxy.server']) def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters {'name': 'catch_errors'}, # already in pipeline {'name': 'proxy_logging', 'after_fn': lambda _: ['catch_errors']}, # not in pipeline, comes after more than one thing {'name': 'container_quotas', 'after_fn': lambda _: ['catch_errors', 'bulk']}] contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, 'required_filters', new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.healthcheck', 'swift.common.middleware.proxy_logging', 'swift.common.middleware.bulk', 'swift.common.middleware.container_quotas', 'swift.common.middleware.tempurl', 'swift.proxy.server']) def _proxy_modify_wsgi_pipeline(self, pipe): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = %s [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ config = config % (pipe,) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) return app def test_gatekeeper_insertion_catch_errors_configured_at_start(self): # catch_errors is configured at start, gatekeeper is not configured, # so gatekeeper should be inserted just after catch_errors pipe = 'catch_errors healthcheck proxy-server' app = self._proxy_modify_wsgi_pipeline(pipe) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', 'swift.common.middleware.healthcheck', 'swift.proxy.server']) def test_gatekeeper_insertion_catch_errors_configured_not_at_start(self): # catch_errors is configured, gatekeeper is not configured, so # gatekeeper should be inserted at start of pipeline pipe = 'healthcheck catch_errors proxy-server' app = self._proxy_modify_wsgi_pipeline(pipe) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.gatekeeper', 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', 'swift.common.middleware.dlo', 'swift.proxy.server']) def test_catch_errors_gatekeeper_configured_not_at_start(self): # catch_errors is configured, gatekeeper is configured, so # no change should be made to pipeline pipe = 'healthcheck catch_errors gatekeeper proxy-server' app = self._proxy_modify_wsgi_pipeline(pipe) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.healthcheck', 'swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.dlo', 'swift.proxy.server']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_daemon.py0000664000175400017540000000664012323703611022175 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # TODO(clayg): Test kill_children signal handlers import os import unittest from getpass import getuser import logging from StringIO import StringIO from test.unit import tmpfile from mock import patch from swift.common import daemon, utils class MyDaemon(daemon.Daemon): def __init__(self, conf): self.conf = conf self.logger = utils.get_logger(None, 'server', log_route='server') MyDaemon.forever_called = False MyDaemon.once_called = False def run_forever(self): MyDaemon.forever_called = True def run_once(self): MyDaemon.once_called = True def run_raise(self): raise OSError def run_quit(self): raise KeyboardInterrupt class TestDaemon(unittest.TestCase): def test_create(self): d = daemon.Daemon({}) self.assertEquals(d.conf, {}) self.assert_(isinstance(d.logger, utils.LogAdapter)) def test_stubs(self): d = daemon.Daemon({}) self.assertRaises(NotImplementedError, d.run_once) self.assertRaises(NotImplementedError, d.run_forever) class TestRunDaemon(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' utils.drop_privileges = lambda *args: None utils.capture_stdio = lambda *args: None def tearDown(self): reload(utils) def test_run(self): d = MyDaemon({}) self.assertFalse(MyDaemon.forever_called) self.assertFalse(MyDaemon.once_called) # test default d.run() self.assertEquals(d.forever_called, True) # test once d.run(once=True) self.assertEquals(d.once_called, True) def test_run_daemon(self): sample_conf = "[my-daemon]\nuser = %s\n" % getuser() with tmpfile(sample_conf) as conf_file: with patch.dict('os.environ', {'TZ': ''}): daemon.run_daemon(MyDaemon, conf_file) self.assertEquals(MyDaemon.forever_called, True) self.assert_(os.environ['TZ'] is not '') daemon.run_daemon(MyDaemon, conf_file, once=True) self.assertEquals(MyDaemon.once_called, True) # test raise in daemon code MyDaemon.run_once = MyDaemon.run_raise self.assertRaises(OSError, daemon.run_daemon, MyDaemon, conf_file, once=True) # test user quit MyDaemon.run_forever = MyDaemon.run_quit sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) logger = utils.get_logger(None, 'server', log_route='server') daemon.run_daemon(MyDaemon, conf_file, logger=logger) self.assert_('user quit' in sio.getvalue().lower()) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_internal_client.py0000664000175400017540000011505112323703614024104 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import json import mock from StringIO import StringIO import unittest from urllib import quote import zlib from eventlet.green import urllib2 from swift.common import internal_client def not_sleep(seconds): pass def unicode_string(start, length): return u''.join([unichr(x) for x in xrange(start, start + length)]) def path_parts(): account = unicode_string(1000, 4) + ' ' + unicode_string(1100, 4) container = unicode_string(2000, 4) + ' ' + unicode_string(2100, 4) obj = unicode_string(3000, 4) + ' ' + unicode_string(3100, 4) return account, container, obj def make_path(account, container=None, obj=None): path = '/v1/%s' % quote(account.encode('utf-8')) if container: path += '/%s' % quote(container.encode('utf-8')) if obj: path += '/%s' % quote(obj.encode('utf-8')) return path class InternalClient(internal_client.InternalClient): def __init__(self): pass class GetMetadataInternalClient(internal_client.InternalClient): def __init__(self, test, path, metadata_prefix, acceptable_statuses): self.test = test self.path = path self.metadata_prefix = metadata_prefix self.acceptable_statuses = acceptable_statuses self.get_metadata_called = 0 self.metadata = 'some_metadata' def _get_metadata(self, path, metadata_prefix, acceptable_statuses=None): self.get_metadata_called += 1 self.test.assertEquals(self.path, path) self.test.assertEquals(self.metadata_prefix, metadata_prefix) self.test.assertEquals(self.acceptable_statuses, acceptable_statuses) return self.metadata class SetMetadataInternalClient(internal_client.InternalClient): def __init__( self, test, path, metadata, metadata_prefix, acceptable_statuses): self.test = test self.path = path self.metadata = metadata self.metadata_prefix = metadata_prefix self.acceptable_statuses = acceptable_statuses self.set_metadata_called = 0 self.metadata = 'some_metadata' def _set_metadata( self, path, metadata, metadata_prefix='', acceptable_statuses=None): self.set_metadata_called += 1 self.test.assertEquals(self.path, path) self.test.assertEquals(self.metadata_prefix, metadata_prefix) self.test.assertEquals(self.metadata, metadata) self.test.assertEquals(self.acceptable_statuses, acceptable_statuses) class IterInternalClient(internal_client.InternalClient): def __init__( self, test, path, marker, end_marker, acceptable_statuses, items): self.test = test self.path = path self.marker = marker self.end_marker = end_marker self.acceptable_statuses = acceptable_statuses self.items = items def _iter_items( self, path, marker='', end_marker='', acceptable_statuses=None): self.test.assertEquals(self.path, path) self.test.assertEquals(self.marker, marker) self.test.assertEquals(self.end_marker, end_marker) self.test.assertEquals(self.acceptable_statuses, acceptable_statuses) for item in self.items: yield item class TestCompressingfileReader(unittest.TestCase): def test_init(self): class CompressObj(object): def __init__(self, test, *args): self.test = test self.args = args def method(self, *args): self.test.assertEquals(self.args, args) return self try: compressobj = CompressObj( self, 9, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) old_compressobj = internal_client.compressobj internal_client.compressobj = compressobj.method f = StringIO('') fobj = internal_client.CompressingFileReader(f) self.assertEquals(f, fobj._f) self.assertEquals(compressobj, fobj._compressor) self.assertEquals(False, fobj.done) self.assertEquals(True, fobj.first) self.assertEquals(0, fobj.crc32) self.assertEquals(0, fobj.total_size) finally: internal_client.compressobj = old_compressobj def test_read(self): exp_data = 'abcdefghijklmnopqrstuvwxyz' fobj = internal_client.CompressingFileReader( StringIO(exp_data), chunk_size=5) data = '' d = zlib.decompressobj(16 + zlib.MAX_WBITS) for chunk in fobj.read(): data += d.decompress(chunk) self.assertEquals(exp_data, data) def test_seek(self): exp_data = 'abcdefghijklmnopqrstuvwxyz' fobj = internal_client.CompressingFileReader( StringIO(exp_data), chunk_size=5) # read a couple of chunks only for _ in range(2): fobj.read() # read whole thing after seek and check data fobj.seek(0) data = '' d = zlib.decompressobj(16 + zlib.MAX_WBITS) for chunk in fobj.read(): data += d.decompress(chunk) self.assertEquals(exp_data, data) def test_seek_not_implemented_exception(self): fobj = internal_client.CompressingFileReader( StringIO(''), chunk_size=5) self.assertRaises(NotImplementedError, fobj.seek, 10) self.assertRaises(NotImplementedError, fobj.seek, 0, 10) class TestInternalClient(unittest.TestCase): def test_init(self): class App(object): def __init__(self, test, conf_path): self.test = test self.conf_path = conf_path self.load_called = 0 def load(self, uri, allow_modify_pipeline=True): self.load_called += 1 self.test.assertEquals(conf_path, uri) self.test.assertFalse(allow_modify_pipeline) return self conf_path = 'some_path' app = App(self, conf_path) old_loadapp = internal_client.loadapp internal_client.loadapp = app.load user_agent = 'some_user_agent' request_tries = 'some_request_tries' try: client = internal_client.InternalClient( conf_path, user_agent, request_tries) finally: internal_client.loadapp = old_loadapp self.assertEquals(1, app.load_called) self.assertEquals(app, client.app) self.assertEquals(user_agent, client.user_agent) self.assertEquals(request_tries, client.request_tries) def test_make_request_sets_user_agent(self): class InternalClient(internal_client.InternalClient): def __init__(self, test): self.test = test self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 1 def fake_app(self, env, start_response): self.test.assertEquals(self.user_agent, env['HTTP_USER_AGENT']) start_response('200 Ok', [('Content-Length', '0')]) return [] client = InternalClient(self) client.make_request('GET', '/', {}, (200,)) def test_make_request_retries(self): class InternalClient(internal_client.InternalClient): def __init__(self, test): self.test = test self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 4 self.tries = 0 self.sleep_called = 0 def fake_app(self, env, start_response): self.tries += 1 if self.tries < self.request_tries: start_response( '500 Internal Server Error', [('Content-Length', '0')]) else: start_response('200 Ok', [('Content-Length', '0')]) return [] def sleep(self, seconds): self.sleep_called += 1 self.test.assertEquals(2 ** (self.sleep_called), seconds) client = InternalClient(self) old_sleep = internal_client.sleep internal_client.sleep = client.sleep try: client.make_request('GET', '/', {}, (200,)) finally: internal_client.sleep = old_sleep self.assertEquals(3, client.sleep_called) self.assertEquals(4, client.tries) def test_make_request_method_path_headers(self): class InternalClient(internal_client.InternalClient): def __init__(self): self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 self.env = None def fake_app(self, env, start_response): self.env = env start_response('200 Ok', [('Content-Length', '0')]) return [] client = InternalClient() for method in 'GET PUT HEAD'.split(): client.make_request(method, '/', {}, (200,)) self.assertEquals(client.env['REQUEST_METHOD'], method) for path in '/one /two/three'.split(): client.make_request('GET', path, {'X-Test': path}, (200,)) self.assertEquals(client.env['PATH_INFO'], path) self.assertEquals(client.env['HTTP_X_TEST'], path) def test_make_request_codes(self): class InternalClient(internal_client.InternalClient): def __init__(self): self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): start_response('200 Ok', [('Content-Length', '0')]) return [] client = InternalClient() try: old_sleep = internal_client.sleep internal_client.sleep = not_sleep client.make_request('GET', '/', {}, (200,)) client.make_request('GET', '/', {}, (2,)) client.make_request('GET', '/', {}, (400, 200)) client.make_request('GET', '/', {}, (400, 2)) try: client.make_request('GET', '/', {}, (400,)) except Exception as err: pass self.assertEquals(200, err.resp.status_int) try: client.make_request('GET', '/', {}, (201,)) except Exception as err: pass self.assertEquals(200, err.resp.status_int) try: client.make_request('GET', '/', {}, (111,)) except Exception as err: self.assertTrue(str(err).startswith('Unexpected response')) else: self.fail("Expected the UnexpectedResponse") finally: internal_client.sleep = old_sleep def test_make_request_calls_fobj_seek_each_try(self): class FileObject(object): def __init__(self, test): self.test = test self.seek_called = 0 def seek(self, offset, whence=0): self.seek_called += 1 self.test.assertEquals(0, offset) self.test.assertEquals(0, whence) class InternalClient(internal_client.InternalClient): def __init__(self): self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): start_response('404 Not Found', [('Content-Length', '0')]) return [] fobj = FileObject(self) client = InternalClient() try: old_sleep = internal_client.sleep internal_client.sleep = not_sleep try: client.make_request('PUT', '/', {}, (2,), fobj) except Exception as err: pass self.assertEquals(404, err.resp.status_int) finally: internal_client.sleep = old_sleep self.assertEquals(client.request_tries, fobj.seek_called) def test_make_request_request_exception(self): class InternalClient(internal_client.InternalClient): def __init__(self): self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): raise Exception() client = InternalClient() try: old_sleep = internal_client.sleep internal_client.sleep = not_sleep self.assertRaises( Exception, client.make_request, 'GET', '/', {}, (2,)) finally: internal_client.sleep = old_sleep def test_get_metadata(self): class Response(object): def __init__(self, headers): self.headers = headers self.status_int = 200 class InternalClient(internal_client.InternalClient): def __init__(self, test, path, resp_headers): self.test = test self.path = path self.resp_headers = resp_headers self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('HEAD', method) self.test.assertEquals(self.path, path) self.test.assertEquals((2,), acceptable_statuses) self.test.assertEquals(None, body_file) return Response(self.resp_headers) path = 'some_path' metadata_prefix = 'some_key-' resp_headers = { '%sone' % (metadata_prefix): '1', '%sTwo' % (metadata_prefix): '2', '%sThree' % (metadata_prefix): '3', 'some_header-four': '4', 'Some_header-five': '5', } exp_metadata = { 'one': '1', 'two': '2', 'three': '3', } client = InternalClient(self, path, resp_headers) metadata = client._get_metadata(path, metadata_prefix) self.assertEquals(exp_metadata, metadata) self.assertEquals(1, client.make_request_called) def test_get_metadata_invalid_status(self): class Response(object): def __init__(self): self.status_int = 404 self.headers = {'some_key': 'some_value'} class InternalClient(internal_client.InternalClient): def __init__(self): pass def make_request(self, *a, **kw): return Response() client = InternalClient() metadata = client._get_metadata('path') self.assertEquals({}, metadata) def test_make_path(self): account, container, obj = path_parts() path = make_path(account, container, obj) c = InternalClient() self.assertEquals(path, c.make_path(account, container, obj)) def test_make_path_exception(self): c = InternalClient() self.assertRaises(ValueError, c.make_path, 'account', None, 'obj') def test_iter_items(self): class Response(object): def __init__(self, status_int, body): self.status_int = status_int self.body = body class InternalClient(internal_client.InternalClient): def __init__(self, test, responses): self.test = test self.responses = responses self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 return self.responses.pop(0) exp_items = [] responses = [Response(200, json.dumps([])), ] items = [] client = InternalClient(self, responses) for item in client._iter_items('/'): items.append(item) self.assertEquals(exp_items, items) exp_items = [] responses = [] for i in xrange(3): data = [ {'name': 'item%02d' % (2 * i)}, {'name': 'item%02d' % (2 * i + 1)}] responses.append(Response(200, json.dumps(data))) exp_items.extend(data) responses.append(Response(204, '')) items = [] client = InternalClient(self, responses) for item in client._iter_items('/'): items.append(item) self.assertEquals(exp_items, items) def test_iter_items_with_markers(self): class Response(object): def __init__(self, status_int, body): self.status_int = status_int self.body = body class InternalClient(internal_client.InternalClient): def __init__(self, test, paths, responses): self.test = test self.paths = paths self.responses = responses def make_request( self, method, path, headers, acceptable_statuses, body_file=None): exp_path = self.paths.pop(0) self.test.assertEquals(exp_path, path) return self.responses.pop(0) paths = [ '/?format=json&marker=start&end_marker=end', '/?format=json&marker=one%C3%A9&end_marker=end', '/?format=json&marker=two&end_marker=end', ] responses = [ Response(200, json.dumps([{'name': 'one\xc3\xa9'}, ])), Response(200, json.dumps([{'name': 'two'}, ])), Response(204, ''), ] items = [] client = InternalClient(self, paths, responses) for item in client._iter_items('/', marker='start', end_marker='end'): items.append(item['name'].encode('utf8')) self.assertEquals('one\xc3\xa9 two'.split(), items) def test_set_metadata(self): class InternalClient(internal_client.InternalClient): def __init__(self, test, path, exp_headers): self.test = test self.path = path self.exp_headers = exp_headers self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('POST', method) self.test.assertEquals(self.path, path) self.test.assertEquals(self.exp_headers, headers) self.test.assertEquals((2,), acceptable_statuses) self.test.assertEquals(None, body_file) path = 'some_path' metadata_prefix = 'some_key-' metadata = { '%sone' % (metadata_prefix): '1', '%stwo' % (metadata_prefix): '2', 'three': '3', } exp_headers = { '%sone' % (metadata_prefix): '1', '%stwo' % (metadata_prefix): '2', '%sthree' % (metadata_prefix): '3', } client = InternalClient(self, path, exp_headers) client._set_metadata(path, metadata, metadata_prefix) self.assertEquals(1, client.make_request_called) def test_iter_containers(self): account, container, obj = path_parts() path = make_path(account) items = '0 1 2'.split() marker = 'some_marker' end_marker = 'some_end_marker' acceptable_statuses = 'some_status_list' client = IterInternalClient( self, path, marker, end_marker, acceptable_statuses, items) ret_items = [] for container in client.iter_containers( account, marker, end_marker, acceptable_statuses=acceptable_statuses): ret_items.append(container) self.assertEquals(items, ret_items) def test_get_account_info(self): class Response(object): def __init__(self, containers, objects): self.headers = { 'x-account-container-count': containers, 'x-account-object-count': objects, } self.status_int = 200 class InternalClient(internal_client.InternalClient): def __init__(self, test, path, resp): self.test = test self.path = path self.resp = resp def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.test.assertEquals('HEAD', method) self.test.assertEquals(self.path, path) self.test.assertEquals({}, headers) self.test.assertEquals((2, 404), acceptable_statuses) self.test.assertEquals(None, body_file) return self.resp account, container, obj = path_parts() path = make_path(account) containers, objects = 10, 100 client = InternalClient(self, path, Response(containers, objects)) info = client.get_account_info(account) self.assertEquals((containers, objects), info) def test_get_account_info_404(self): class Response(object): def __init__(self): self.headers = { 'x-account-container-count': 10, 'x-account-object-count': 100, } self.status_int = 404 class InternalClient(internal_client.InternalClient): def __init__(self): pass def make_path(self, *a, **kw): return 'some_path' def make_request(self, *a, **kw): return Response() client = InternalClient() info = client.get_account_info('some_account') self.assertEquals((0, 0), info) def test_get_account_metadata(self): account, container, obj = path_parts() path = make_path(account) acceptable_statuses = 'some_status_list' metadata_prefix = 'some_metadata_prefix' client = GetMetadataInternalClient( self, path, metadata_prefix, acceptable_statuses) metadata = client.get_account_metadata( account, metadata_prefix, acceptable_statuses) self.assertEquals(client.metadata, metadata) self.assertEquals(1, client.get_metadata_called) def test_set_account_metadata(self): account, container, obj = path_parts() path = make_path(account) metadata = 'some_metadata' metadata_prefix = 'some_metadata_prefix' acceptable_statuses = 'some_status_list' client = SetMetadataInternalClient( self, path, metadata, metadata_prefix, acceptable_statuses) client.set_account_metadata( account, metadata, metadata_prefix, acceptable_statuses) self.assertEquals(1, client.set_metadata_called) def test_container_exists(self): class Response(object): def __init__(self, status_int): self.status_int = status_int class InternalClient(internal_client.InternalClient): def __init__(self, test, path, resp): self.test = test self.path = path self.make_request_called = 0 self.resp = resp def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('HEAD', method) self.test.assertEquals(self.path, path) self.test.assertEquals({}, headers) self.test.assertEquals((2, 404), acceptable_statuses) self.test.assertEquals(None, body_file) return self.resp account, container, obj = path_parts() path = make_path(account, container) client = InternalClient(self, path, Response(200)) self.assertEquals(True, client.container_exists(account, container)) self.assertEquals(1, client.make_request_called) client = InternalClient(self, path, Response(404)) self.assertEquals(False, client.container_exists(account, container)) self.assertEquals(1, client.make_request_called) def test_create_container(self): class InternalClient(internal_client.InternalClient): def __init__(self, test, path, headers): self.test = test self.path = path self.headers = headers self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('PUT', method) self.test.assertEquals(self.path, path) self.test.assertEquals(self.headers, headers) self.test.assertEquals((2,), acceptable_statuses) self.test.assertEquals(None, body_file) account, container, obj = path_parts() path = make_path(account, container) headers = 'some_headers' client = InternalClient(self, path, headers) client.create_container(account, container, headers) self.assertEquals(1, client.make_request_called) def test_delete_container(self): class InternalClient(internal_client.InternalClient): def __init__(self, test, path): self.test = test self.path = path self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('DELETE', method) self.test.assertEquals(self.path, path) self.test.assertEquals({}, headers) self.test.assertEquals((2, 404), acceptable_statuses) self.test.assertEquals(None, body_file) account, container, obj = path_parts() path = make_path(account, container) client = InternalClient(self, path) client.delete_container(account, container) self.assertEquals(1, client.make_request_called) def test_get_container_metadata(self): account, container, obj = path_parts() path = make_path(account, container) metadata_prefix = 'some_metadata_prefix' acceptable_statuses = 'some_status_list' client = GetMetadataInternalClient( self, path, metadata_prefix, acceptable_statuses) metadata = client.get_container_metadata( account, container, metadata_prefix, acceptable_statuses) self.assertEquals(client.metadata, metadata) self.assertEquals(1, client.get_metadata_called) def test_iter_objects(self): account, container, obj = path_parts() path = make_path(account, container) marker = 'some_maker' end_marker = 'some_end_marker' acceptable_statuses = 'some_status_list' items = '0 1 2'.split() client = IterInternalClient( self, path, marker, end_marker, acceptable_statuses, items) ret_items = [] for obj in client.iter_objects( account, container, marker, end_marker, acceptable_statuses): ret_items.append(obj) self.assertEquals(items, ret_items) def test_set_container_metadata(self): account, container, obj = path_parts() path = make_path(account, container) metadata = 'some_metadata' metadata_prefix = 'some_metadata_prefix' acceptable_statuses = 'some_status_list' client = SetMetadataInternalClient( self, path, metadata, metadata_prefix, acceptable_statuses) client.set_container_metadata( account, container, metadata, metadata_prefix, acceptable_statuses) self.assertEquals(1, client.set_metadata_called) def test_delete_object(self): class InternalClient(internal_client.InternalClient): def __init__(self, test, path): self.test = test self.path = path self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals('DELETE', method) self.test.assertEquals(self.path, path) self.test.assertEquals({}, headers) self.test.assertEquals((2, 404), acceptable_statuses) self.test.assertEquals(None, body_file) account, container, obj = path_parts() path = make_path(account, container, obj) client = InternalClient(self, path) client.delete_object(account, container, obj) self.assertEquals(1, client.make_request_called) def test_get_object_metadata(self): account, container, obj = path_parts() path = make_path(account, container, obj) metadata_prefix = 'some_metadata_prefix' acceptable_statuses = 'some_status_list' client = GetMetadataInternalClient( self, path, metadata_prefix, acceptable_statuses) metadata = client.get_object_metadata( account, container, obj, metadata_prefix, acceptable_statuses) self.assertEquals(client.metadata, metadata) self.assertEquals(1, client.get_metadata_called) def test_iter_object_lines(self): class InternalClient(internal_client.InternalClient): def __init__(self, lines): self.lines = lines self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): start_response('200 Ok', [('Content-Length', '0')]) return ['%s\n' % x for x in self.lines] lines = 'line1 line2 line3'.split() client = InternalClient(lines) ret_lines = [] for line in client.iter_object_lines('account', 'container', 'object'): ret_lines.append(line) self.assertEquals(lines, ret_lines) def test_iter_object_lines_compressed_object(self): class InternalClient(internal_client.InternalClient): def __init__(self, lines): self.lines = lines self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): start_response('200 Ok', [('Content-Length', '0')]) return internal_client.CompressingFileReader( StringIO('\n'.join(self.lines))) lines = 'line1 line2 line3'.split() client = InternalClient(lines) ret_lines = [] for line in client.iter_object_lines( 'account', 'container', 'object.gz'): ret_lines.append(line) self.assertEquals(lines, ret_lines) def test_iter_object_lines_404(self): class InternalClient(internal_client.InternalClient): def __init__(self): self.app = self.fake_app self.user_agent = 'some_agent' self.request_tries = 3 def fake_app(self, env, start_response): start_response('404 Not Found', []) return ['one\ntwo\nthree'] client = InternalClient() lines = [] for line in client.iter_object_lines( 'some_account', 'some_container', 'some_object', acceptable_statuses=(2, 404)): lines.append(line) self.assertEquals([], lines) def test_set_object_metadata(self): account, container, obj = path_parts() path = make_path(account, container, obj) metadata = 'some_metadata' metadata_prefix = 'some_metadata_prefix' acceptable_statuses = 'some_status_list' client = SetMetadataInternalClient( self, path, metadata, metadata_prefix, acceptable_statuses) client.set_object_metadata( account, container, obj, metadata, metadata_prefix, acceptable_statuses) self.assertEquals(1, client.set_metadata_called) def test_upload_object(self): class InternalClient(internal_client.InternalClient): def __init__(self, test, path, headers, fobj): self.test = test self.path = path self.headers = headers self.fobj = fobj self.make_request_called = 0 def make_request( self, method, path, headers, acceptable_statuses, body_file=None): self.make_request_called += 1 self.test.assertEquals(self.path, path) exp_headers = dict(self.headers) exp_headers['Transfer-Encoding'] = 'chunked' self.test.assertEquals(exp_headers, headers) self.test.assertEquals(self.fobj, fobj) fobj = 'some_fobj' account, container, obj = path_parts() path = make_path(account, container, obj) headers = {'key': 'value'} client = InternalClient(self, path, headers, fobj) client.upload_object(fobj, account, container, obj, headers) self.assertEquals(1, client.make_request_called) class TestGetAuth(unittest.TestCase): @mock.patch('eventlet.green.urllib2.urlopen') @mock.patch('eventlet.green.urllib2.Request') def test_ok(self, request, urlopen): def getheader(name): d = {'X-Storage-Url': 'url', 'X-Auth-Token': 'token'} return d.get(name) urlopen.return_value.info.return_value.getheader = getheader url, token = internal_client.get_auth( 'http://127.0.0.1', 'user', 'key') self.assertEqual(url, "url") self.assertEqual(token, "token") request.assert_called_with('http://127.0.0.1') request.return_value.add_header.assert_any_call('X-Auth-User', 'user') request.return_value.add_header.assert_any_call('X-Auth-Key', 'key') def test_invalid_version(self): self.assertRaises(SystemExit, internal_client.get_auth, 'http://127.0.0.1', 'user', 'key', auth_version=2.0) class TestSimpleClient(unittest.TestCase): @mock.patch('eventlet.green.urllib2.urlopen') @mock.patch('eventlet.green.urllib2.Request') def test_get(self, request, urlopen): # basic GET request, only url as kwarg request.return_value.get_type.return_value = "http" urlopen.return_value.read.return_value = '' sc = internal_client.SimpleClient(url='http://127.0.0.1') retval = sc.retry_request('GET') request.assert_called_with('http://127.0.0.1?format=json', headers={}, data=None) self.assertEqual([None, None], retval) self.assertEqual('GET', request.return_value.get_method()) # Check if JSON is decoded urlopen.return_value.read.return_value = '{}' retval = sc.retry_request('GET') self.assertEqual([None, {}], retval) # same as above, now with token sc = internal_client.SimpleClient(url='http://127.0.0.1', token='token') retval = sc.retry_request('GET') request.assert_called_with('http://127.0.0.1?format=json', headers={'X-Auth-Token': 'token'}, data=None) self.assertEqual([None, {}], retval) # same as above, now with prefix sc = internal_client.SimpleClient(url='http://127.0.0.1', token='token') retval = sc.retry_request('GET', prefix="pre_") request.assert_called_with('http://127.0.0.1?format=json&prefix=pre_', headers={'X-Auth-Token': 'token'}, data=None) self.assertEqual([None, {}], retval) # same as above, now with container name retval = sc.retry_request('GET', container='cont') request.assert_called_with('http://127.0.0.1/cont?format=json', headers={'X-Auth-Token': 'token'}, data=None) self.assertEqual([None, {}], retval) # same as above, now with object name retval = sc.retry_request('GET', container='cont', name='obj') request.assert_called_with('http://127.0.0.1/cont/obj?format=json', headers={'X-Auth-Token': 'token'}, data=None) self.assertEqual([None, {}], retval) @mock.patch('eventlet.green.urllib2.urlopen') @mock.patch('eventlet.green.urllib2.Request') def test_get_with_retries_all_failed(self, request, urlopen): # Simulate a failing request, ensure retries done request.return_value.get_type.return_value = "http" request.side_effect = urllib2.URLError('') urlopen.return_value.read.return_value = '' sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) self.assertRaises(urllib2.URLError, sc.retry_request, 'GET') self.assertEqual(request.call_count, 2) @mock.patch('eventlet.green.urllib2.urlopen') @mock.patch('eventlet.green.urllib2.Request') def test_get_with_retries(self, request, urlopen): # First request fails, retry successful request.return_value.get_type.return_value = "http" urlopen.return_value.read.return_value = '' req = urllib2.Request('http://127.0.0.1', method='GET') request.side_effect = [urllib2.URLError(''), req] sc = internal_client.SimpleClient(url='http://127.0.0.1', retries=1) retval = sc.retry_request('GET') self.assertEqual(request.call_count, 3) request.assert_called_with('http://127.0.0.1?format=json', data=None, headers={'X-Auth-Token': 'token'}) self.assertEqual([None, None], retval) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_container_sync_realms.py0000664000175400017540000001604012323703611025306 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import unittest import uuid from swift.common.container_sync_realms import ContainerSyncRealms from test.unit import FakeLogger, temptree class TestUtils(unittest.TestCase): def test_no_file_there(self): unique = uuid.uuid4().hex logger = FakeLogger() csr = ContainerSyncRealms(unique, logger) self.assertEqual( logger.lines_dict, {'debug': [ "Could not load '%s': [Errno 2] No such file or directory: " "'%s'" % (unique, unique)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), []) def test_os_error(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) os.chmod(tempdir, 0) csr = ContainerSyncRealms(fpath, logger) try: self.assertEqual( logger.lines_dict, {'error': [ "Could not load '%s': [Errno 13] Permission denied: " "'%s'" % (fpath, fpath)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), []) finally: os.chmod(tempdir, 0700) def test_empty(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), []) def test_error_parsing(self): fname = 'container-sync-realms.conf' fcontents = 'invalid' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( logger.lines_dict, {'error': [ "Could not load '%s': File contains no section headers.\n" "file: %s, line: 1\n" "'invalid'" % (fpath, fpath)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), []) def test_one_realm(self): fname = 'container-sync-realms.conf' fcontents = ''' [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), ['US']) self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') self.assertEqual(csr.key2('US'), None) self.assertEqual(csr.clusters('US'), ['DFW1']) self.assertEqual( csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') def test_two_realms_and_change_a_default(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = 60 [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ [UK] key = e9569809dc8b4951accc1487aa788012 key2 = f6351bd1cc36413baa43f7ba1b45e51d cluster_lon3 = http://lon3.host/v1/ ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 60) self.assertEqual(sorted(csr.realms()), ['UK', 'US']) self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') self.assertEqual(csr.key2('US'), None) self.assertEqual(csr.clusters('US'), ['DFW1']) self.assertEqual( csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012') self.assertEqual( csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d') self.assertEqual(csr.clusters('UK'), ['LON3']) self.assertEqual( csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/') def test_empty_realm(self): fname = 'container-sync-realms.conf' fcontents = ''' [US] ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), ['US']) self.assertEqual(csr.key('US'), None) self.assertEqual(csr.key2('US'), None) self.assertEqual(csr.clusters('US'), []) self.assertEqual(csr.endpoint('US', 'JUST_TESTING'), None) def test_bad_mtime_check_interval(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = invalid ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( logger.lines_dict, {'error': [ "Error in '%s' with mtime_check_interval: invalid literal " "for int() with base 10: 'invalid'" % fpath]}) self.assertEqual(csr.mtime_check_interval, 300) def test_get_sig(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( csr.get_sig( 'GET', '/some/path', '1387212345.67890', 'my_nonce', 'realm_key', 'user_key'), '5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_utils.py0000664000175400017540000035664712323703614022114 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.common.utils""" from test.unit import temptree import ctypes import errno import eventlet import eventlet.event import grp import logging import os import random import re import socket import sys import json from textwrap import dedent import tempfile import threading import time import traceback import unittest import fcntl import shutil from contextlib import nested from Queue import Queue, Empty from getpass import getuser from shutil import rmtree from StringIO import StringIO from functools import partial from tempfile import TemporaryFile, NamedTemporaryFile, mkdtemp from netifaces import AF_INET6 from mock import MagicMock, patch from swift.common.exceptions import (Timeout, MessageTimeout, ConnectionTimeout, LockTimeout, ReplicationLockTimeout) from swift.common import utils from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import Response from test.unit import FakeLogger class MockOs(object): def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]): self.closed_fds = [] for func in pass_funcs: setattr(self, func, self.pass_func) self.called_funcs = {} for func in called_funcs: c_func = partial(self.called_func, func) setattr(self, func, c_func) for func in raise_funcs: r_func = partial(self.raise_func, func) setattr(self, func, r_func) def pass_func(self, *args, **kwargs): pass setgroups = chdir = setsid = setgid = setuid = umask = pass_func def called_func(self, name, *args, **kwargs): self.called_funcs[name] = True def raise_func(self, name, *args, **kwargs): self.called_funcs[name] = True raise OSError() def dup2(self, source, target): self.closed_fds.append(target) def geteuid(self): '''Pretend we are running as root.''' return 0 def __getattr__(self, name): # I only over-ride portions of the os module try: return object.__getattr__(self, name) except AttributeError: return getattr(os, name) class MockUdpSocket(object): def __init__(self): self.sent = [] def sendto(self, data, target): self.sent.append((data, target)) def close(self): pass class MockSys(object): def __init__(self): self.stdin = TemporaryFile('w') self.stdout = TemporaryFile('r') self.stderr = TemporaryFile('r') self.__stderr__ = self.stderr self.stdio_fds = [self.stdin.fileno(), self.stdout.fileno(), self.stderr.fileno()] @property def version_info(self): return sys.version_info def reset_loggers(): if hasattr(utils.get_logger, 'handler4logger'): for logger, handler in utils.get_logger.handler4logger.items(): logger.thread_locals = (None, None) logger.removeHandler(handler) delattr(utils.get_logger, 'handler4logger') if hasattr(utils.get_logger, 'console_handler4logger'): for logger, h in utils.get_logger.console_handler4logger.items(): logger.thread_locals = (None, None) logger.removeHandler(h) delattr(utils.get_logger, 'console_handler4logger') class TestUtils(unittest.TestCase): """Tests for swift.common.utils """ def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' def test_lock_path(self): tmpdir = mkdtemp() try: with utils.lock_path(tmpdir, 0.1): exc = None success = False try: with utils.lock_path(tmpdir, 0.1): success = True except LockTimeout as err: exc = err self.assertTrue(exc is not None) self.assertTrue(not success) finally: shutil.rmtree(tmpdir) def test_lock_path_class(self): tmpdir = mkdtemp() try: with utils.lock_path(tmpdir, 0.1, ReplicationLockTimeout): exc = None exc2 = None success = False try: with utils.lock_path(tmpdir, 0.1, ReplicationLockTimeout): success = True except ReplicationLockTimeout as err: exc = err except LockTimeout as err: exc2 = err self.assertTrue(exc is not None) self.assertTrue(exc2 is None) self.assertTrue(not success) exc = None exc2 = None success = False try: with utils.lock_path(tmpdir, 0.1): success = True except ReplicationLockTimeout as err: exc = err except LockTimeout as err: exc2 = err self.assertTrue(exc is None) self.assertTrue(exc2 is not None) self.assertTrue(not success) finally: shutil.rmtree(tmpdir) def test_normalize_timestamp(self): # Test swift.common.utils.normalize_timestamp self.assertEquals(utils.normalize_timestamp('1253327593.48174'), "1253327593.48174") self.assertEquals(utils.normalize_timestamp(1253327593.48174), "1253327593.48174") self.assertEquals(utils.normalize_timestamp('1253327593.48'), "1253327593.48000") self.assertEquals(utils.normalize_timestamp(1253327593.48), "1253327593.48000") self.assertEquals(utils.normalize_timestamp('253327593.48'), "0253327593.48000") self.assertEquals(utils.normalize_timestamp(253327593.48), "0253327593.48000") self.assertEquals(utils.normalize_timestamp('1253327593'), "1253327593.00000") self.assertEquals(utils.normalize_timestamp(1253327593), "1253327593.00000") self.assertRaises(ValueError, utils.normalize_timestamp, '') self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') def test_normalize_delete_at_timestamp(self): self.assertEquals( utils.normalize_delete_at_timestamp(1253327593), '1253327593') self.assertEquals( utils.normalize_delete_at_timestamp(1253327593.67890), '1253327593') self.assertEquals( utils.normalize_delete_at_timestamp('1253327593'), '1253327593') self.assertEquals( utils.normalize_delete_at_timestamp('1253327593.67890'), '1253327593') self.assertEquals( utils.normalize_delete_at_timestamp(-1253327593), '0000000000') self.assertEquals( utils.normalize_delete_at_timestamp(-1253327593.67890), '0000000000') self.assertEquals( utils.normalize_delete_at_timestamp('-1253327593'), '0000000000') self.assertEquals( utils.normalize_delete_at_timestamp('-1253327593.67890'), '0000000000') self.assertEquals( utils.normalize_delete_at_timestamp(71253327593), '9999999999') self.assertEquals( utils.normalize_delete_at_timestamp(71253327593.67890), '9999999999') self.assertEquals( utils.normalize_delete_at_timestamp('71253327593'), '9999999999') self.assertEquals( utils.normalize_delete_at_timestamp('71253327593.67890'), '9999999999') self.assertRaises(ValueError, utils.normalize_timestamp, '') self.assertRaises(ValueError, utils.normalize_timestamp, 'abc') def test_backwards(self): # Test swift.common.utils.backward # The lines are designed so that the function would encounter # all of the boundary conditions and typical conditions. # Block boundaries are marked with '<>' characters blocksize = 25 lines = ['123456789x12345678><123456789\n', # block larger than rest '123456789x123>\n', # block ends just before \n character '123423456789\n', '123456789x\n', # block ends at the end of line '<123456789x123456789x123\n', '<6789x123\n', # block ends at the beginning of the line '6789x1234\n', '1234><234\n', # block ends typically in the middle of line '123456789x123456789\n'] with TemporaryFile('r+w') as f: for line in lines: f.write(line) count = len(lines) - 1 for line in utils.backward(f, blocksize): self.assertEquals(line, lines[count].split('\n')[0]) count -= 1 # Empty file case with TemporaryFile('r') as f: self.assertEquals([], list(utils.backward(f))) def test_mkdirs(self): testdir_base = mkdtemp() testroot = os.path.join(testdir_base, 'mkdirs') try: self.assert_(not os.path.exists(testroot)) utils.mkdirs(testroot) self.assert_(os.path.exists(testroot)) utils.mkdirs(testroot) self.assert_(os.path.exists(testroot)) rmtree(testroot, ignore_errors=1) testdir = os.path.join(testroot, 'one/two/three') self.assert_(not os.path.exists(testdir)) utils.mkdirs(testdir) self.assert_(os.path.exists(testdir)) utils.mkdirs(testdir) self.assert_(os.path.exists(testdir)) rmtree(testroot, ignore_errors=1) open(testroot, 'wb').close() self.assert_(not os.path.exists(testdir)) self.assertRaises(OSError, utils.mkdirs, testdir) os.unlink(testroot) finally: rmtree(testdir_base) def test_split_path(self): # Test swift.common.utils.split_account_path self.assertRaises(ValueError, utils.split_path, '') self.assertRaises(ValueError, utils.split_path, '/') self.assertRaises(ValueError, utils.split_path, '//') self.assertEquals(utils.split_path('/a'), ['a']) self.assertRaises(ValueError, utils.split_path, '//a') self.assertEquals(utils.split_path('/a/'), ['a']) self.assertRaises(ValueError, utils.split_path, '/a/c') self.assertRaises(ValueError, utils.split_path, '//c') self.assertRaises(ValueError, utils.split_path, '/a/c/') self.assertRaises(ValueError, utils.split_path, '/a//') self.assertRaises(ValueError, utils.split_path, '/a', 2) self.assertRaises(ValueError, utils.split_path, '/a', 2, 3) self.assertRaises(ValueError, utils.split_path, '/a', 2, 3, True) self.assertEquals(utils.split_path('/a/c', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/o', 3), ['a', 'c', 'o']) self.assertRaises(ValueError, utils.split_path, '/a/c/o/r', 3, 3) self.assertEquals(utils.split_path('/a/c/o/r', 3, 3, True), ['a', 'c', 'o/r']) self.assertEquals(utils.split_path('/a/c', 2, 3, True), ['a', 'c', None]) self.assertRaises(ValueError, utils.split_path, '/a', 5, 4) self.assertEquals(utils.split_path('/a/c/', 2), ['a', 'c']) self.assertEquals(utils.split_path('/a/c/', 2, 3), ['a', 'c', '']) try: utils.split_path('o\nn e', 2) except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') try: utils.split_path('o\nn e', 2, 3, True) except ValueError as err: self.assertEquals(str(err), 'Invalid path: o%0An%20e') def test_validate_device_partition(self): # Test swift.common.utils.validate_device_partition utils.validate_device_partition('foo', 'bar') self.assertRaises(ValueError, utils.validate_device_partition, '', '') self.assertRaises(ValueError, utils.validate_device_partition, '', 'foo') self.assertRaises(ValueError, utils.validate_device_partition, 'foo', '') self.assertRaises(ValueError, utils.validate_device_partition, 'foo/bar', 'foo') self.assertRaises(ValueError, utils.validate_device_partition, 'foo', 'foo/bar') self.assertRaises(ValueError, utils.validate_device_partition, '.', 'foo') self.assertRaises(ValueError, utils.validate_device_partition, '..', 'foo') self.assertRaises(ValueError, utils.validate_device_partition, 'foo', '.') self.assertRaises(ValueError, utils.validate_device_partition, 'foo', '..') try: utils.validate_device_partition('o\nn e', 'foo') except ValueError as err: self.assertEquals(str(err), 'Invalid device: o%0An%20e') try: utils.validate_device_partition('foo', 'o\nn e') except ValueError as err: self.assertEquals(str(err), 'Invalid partition: o%0An%20e') def test_NullLogger(self): # Test swift.common.utils.NullLogger sio = StringIO() nl = utils.NullLogger() nl.write('test') self.assertEquals(sio.getvalue(), '') def test_LoggerFileObject(self): orig_stdout = sys.stdout orig_stderr = sys.stderr sio = StringIO() handler = logging.StreamHandler(sio) logger = logging.getLogger() logger.addHandler(handler) lfo = utils.LoggerFileObject(logger) print 'test1' self.assertEquals(sio.getvalue(), '') sys.stdout = lfo print 'test2' self.assertEquals(sio.getvalue(), 'STDOUT: test2\n') sys.stderr = lfo print >> sys.stderr, 'test4' self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n') sys.stdout = orig_stdout print 'test5' self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n') print >> sys.stderr, 'test6' self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' 'STDOUT: test6\n') sys.stderr = orig_stderr print 'test8' self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' 'STDOUT: test6\n') lfo.writelines(['a', 'b', 'c']) self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' 'STDOUT: test6\nSTDOUT: a#012b#012c\n') lfo.close() lfo.write('d') self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' 'STDOUT: test6\nSTDOUT: a#012b#012c\nSTDOUT: d\n') lfo.flush() self.assertEquals(sio.getvalue(), 'STDOUT: test2\nSTDOUT: test4\n' 'STDOUT: test6\nSTDOUT: a#012b#012c\nSTDOUT: d\n') got_exc = False try: for line in lfo: pass except Exception: got_exc = True self.assert_(got_exc) got_exc = False try: for line in lfo.xreadlines(): pass except Exception: got_exc = True self.assert_(got_exc) self.assertRaises(IOError, lfo.read) self.assertRaises(IOError, lfo.read, 1024) self.assertRaises(IOError, lfo.readline) self.assertRaises(IOError, lfo.readline, 1024) lfo.tell() def test_parse_options(self): # Get a file that is definitely on disk with NamedTemporaryFile() as f: conf_file = f.name conf, options = utils.parse_options(test_args=[conf_file]) self.assertEquals(conf, conf_file) # assert defaults self.assertEquals(options['verbose'], False) self.assert_('once' not in options) # assert verbose as option conf, options = utils.parse_options(test_args=[conf_file, '-v']) self.assertEquals(options['verbose'], True) # check once option conf, options = utils.parse_options(test_args=[conf_file], once=True) self.assertEquals(options['once'], False) test_args = [conf_file, '--once'] conf, options = utils.parse_options(test_args=test_args, once=True) self.assertEquals(options['once'], True) # check options as arg parsing test_args = [conf_file, 'once', 'plugin_name', 'verbose'] conf, options = utils.parse_options(test_args=test_args, once=True) self.assertEquals(options['verbose'], True) self.assertEquals(options['once'], True) self.assertEquals(options['extra_args'], ['plugin_name']) def test_parse_options_errors(self): orig_stdout = sys.stdout orig_stderr = sys.stderr stdo = StringIO() stde = StringIO() utils.sys.stdout = stdo utils.sys.stderr = stde self.assertRaises(SystemExit, utils.parse_options, once=True, test_args=[]) self.assert_('missing config' in stdo.getvalue()) # verify conf file must exist, context manager will delete temp file with NamedTemporaryFile() as f: conf_file = f.name self.assertRaises(SystemExit, utils.parse_options, once=True, test_args=[conf_file]) self.assert_('unable to locate' in stdo.getvalue()) # reset stdio utils.sys.stdout = orig_stdout utils.sys.stderr = orig_stderr def test_dump_recon_cache(self): testdir_base = mkdtemp() testcache_file = os.path.join(testdir_base, 'cache.recon') logger = utils.get_logger(None, 'server', log_route='server') try: submit_dict = {'key1': {'value1': 1, 'value2': 2}} utils.dump_recon_cache(submit_dict, testcache_file, logger) fd = open(testcache_file) file_dict = json.loads(fd.readline()) fd.close() self.assertEquals(submit_dict, file_dict) # Use a nested entry submit_dict = {'key1': {'key2': {'value1': 1, 'value2': 2}}} result_dict = {'key1': {'key2': {'value1': 1, 'value2': 2}, 'value1': 1, 'value2': 2}} utils.dump_recon_cache(submit_dict, testcache_file, logger) fd = open(testcache_file) file_dict = json.loads(fd.readline()) fd.close() self.assertEquals(result_dict, file_dict) finally: rmtree(testdir_base) def test_get_logger(self): sio = StringIO() logger = logging.getLogger('server') logger.addHandler(logging.StreamHandler(sio)) logger = utils.get_logger(None, 'server', log_route='server') logger.warn('test1') self.assertEquals(sio.getvalue(), 'test1\n') logger.debug('test2') self.assertEquals(sio.getvalue(), 'test1\n') logger = utils.get_logger({'log_level': 'DEBUG'}, 'server', log_route='server') logger.debug('test3') self.assertEquals(sio.getvalue(), 'test1\ntest3\n') # Doesn't really test that the log facility is truly being used all the # way to syslog; but exercises the code. logger = utils.get_logger({'log_facility': 'LOG_LOCAL3'}, 'server', log_route='server') logger.warn('test4') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') # make sure debug doesn't log by default logger.debug('test5') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\n') # make sure notice lvl logs by default logger.notice('test6') self.assertEquals(sio.getvalue(), 'test1\ntest3\ntest4\ntest6\n') def test_get_logger_sysloghandler_plumbing(self): orig_sysloghandler = utils.SysLogHandler syslog_handler_args = [] def syslog_handler_catcher(*args, **kwargs): syslog_handler_args.append((args, kwargs)) return orig_sysloghandler(*args, **kwargs) syslog_handler_catcher.LOG_LOCAL0 = orig_sysloghandler.LOG_LOCAL0 syslog_handler_catcher.LOG_LOCAL3 = orig_sysloghandler.LOG_LOCAL3 try: utils.SysLogHandler = syslog_handler_catcher utils.get_logger({ 'log_facility': 'LOG_LOCAL3', }, 'server', log_route='server') expected_args = [((), {'address': '/dev/log', 'facility': orig_sysloghandler.LOG_LOCAL3})] if not os.path.exists('/dev/log') or \ os.path.isfile('/dev/log') or \ os.path.isdir('/dev/log'): # Since socket on OSX is in /var/run/syslog, there will be # a fallback to UDP. expected_args.append( ((), {'facility': orig_sysloghandler.LOG_LOCAL3})) self.assertEquals(expected_args, syslog_handler_args) syslog_handler_args = [] utils.get_logger({ 'log_facility': 'LOG_LOCAL3', 'log_address': '/foo/bar', }, 'server', log_route='server') self.assertEquals([ ((), {'address': '/foo/bar', 'facility': orig_sysloghandler.LOG_LOCAL3}), # Second call is because /foo/bar didn't exist (and wasn't a # UNIX domain socket). ((), {'facility': orig_sysloghandler.LOG_LOCAL3})], syslog_handler_args) # Using UDP with default port syslog_handler_args = [] utils.get_logger({ 'log_udp_host': 'syslog.funtimes.com', }, 'server', log_route='server') self.assertEquals([ ((), {'address': ('syslog.funtimes.com', logging.handlers.SYSLOG_UDP_PORT), 'facility': orig_sysloghandler.LOG_LOCAL0})], syslog_handler_args) # Using UDP with non-default port syslog_handler_args = [] utils.get_logger({ 'log_udp_host': 'syslog.funtimes.com', 'log_udp_port': '2123', }, 'server', log_route='server') self.assertEquals([ ((), {'address': ('syslog.funtimes.com', 2123), 'facility': orig_sysloghandler.LOG_LOCAL0})], syslog_handler_args) finally: utils.SysLogHandler = orig_sysloghandler def test_clean_logger_exception(self): # setup stream logging sio = StringIO() logger = utils.get_logger(None) handler = logging.StreamHandler(sio) logger.logger.addHandler(handler) def strip_value(sio): v = sio.getvalue() sio.truncate(0) return v def log_exception(exc): try: raise exc except (Exception, Timeout): logger.exception('blah') try: # establish base case self.assertEquals(strip_value(sio), '') logger.info('test') self.assertEquals(strip_value(sio), 'test\n') self.assertEquals(strip_value(sio), '') logger.info('test') logger.info('test') self.assertEquals(strip_value(sio), 'test\ntest\n') self.assertEquals(strip_value(sio), '') # test OSError for en in (errno.EIO, errno.ENOSPC): log_exception(OSError(en, 'my %s error message' % en)) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('my %s error message' % en in log_msg) # unfiltered log_exception(OSError()) self.assert_('Traceback' in strip_value(sio)) # test socket.error log_exception(socket.error(errno.ECONNREFUSED, 'my error message')) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('errno.ECONNREFUSED message test' not in log_msg) self.assert_('Connection refused' in log_msg) log_exception(socket.error(errno.EHOSTUNREACH, 'my error message')) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('my error message' not in log_msg) self.assert_('Host unreachable' in log_msg) log_exception(socket.error(errno.ETIMEDOUT, 'my error message')) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('my error message' not in log_msg) self.assert_('Connection timeout' in log_msg) # unfiltered log_exception(socket.error(0, 'my error message')) log_msg = strip_value(sio) self.assert_('Traceback' in log_msg) self.assert_('my error message' in log_msg) # test eventlet.Timeout connection_timeout = ConnectionTimeout(42, 'my error message') log_exception(connection_timeout) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('ConnectionTimeout' in log_msg) self.assert_('(42s)' in log_msg) self.assert_('my error message' not in log_msg) connection_timeout.cancel() message_timeout = MessageTimeout(42, 'my error message') log_exception(message_timeout) log_msg = strip_value(sio) self.assert_('Traceback' not in log_msg) self.assert_('MessageTimeout' in log_msg) self.assert_('(42s)' in log_msg) self.assert_('my error message' in log_msg) message_timeout.cancel() # test unhandled log_exception(Exception('my error message')) log_msg = strip_value(sio) self.assert_('Traceback' in log_msg) self.assert_('my error message' in log_msg) finally: logger.logger.removeHandler(handler) reset_loggers() def test_swift_log_formatter(self): # setup stream logging sio = StringIO() logger = utils.get_logger(None) handler = logging.StreamHandler(sio) handler.setFormatter(utils.SwiftLogFormatter()) logger.logger.addHandler(handler) def strip_value(sio): v = sio.getvalue() sio.truncate(0) return v try: self.assertFalse(logger.txn_id) logger.error('my error message') log_msg = strip_value(sio) self.assert_('my error message' in log_msg) self.assert_('txn' not in log_msg) logger.txn_id = '12345' logger.error('test') log_msg = strip_value(sio) self.assert_('txn' in log_msg) self.assert_('12345' in log_msg) # test no txn on info message self.assertEquals(logger.txn_id, '12345') logger.info('test') log_msg = strip_value(sio) self.assert_('txn' not in log_msg) self.assert_('12345' not in log_msg) # test txn already in message self.assertEquals(logger.txn_id, '12345') logger.warn('test 12345 test') self.assertEquals(strip_value(sio), 'test 12345 test\n') # Test multi line collapsing logger.error('my\nerror\nmessage') log_msg = strip_value(sio) self.assert_('my#012error#012message' in log_msg) # test client_ip self.assertFalse(logger.client_ip) logger.error('my error message') log_msg = strip_value(sio) self.assert_('my error message' in log_msg) self.assert_('client_ip' not in log_msg) logger.client_ip = '1.2.3.4' logger.error('test') log_msg = strip_value(sio) self.assert_('client_ip' in log_msg) self.assert_('1.2.3.4' in log_msg) # test no client_ip on info message self.assertEquals(logger.client_ip, '1.2.3.4') logger.info('test') log_msg = strip_value(sio) self.assert_('client_ip' not in log_msg) self.assert_('1.2.3.4' not in log_msg) # test client_ip (and txn) already in message self.assertEquals(logger.client_ip, '1.2.3.4') logger.warn('test 1.2.3.4 test 12345') self.assertEquals(strip_value(sio), 'test 1.2.3.4 test 12345\n') finally: logger.logger.removeHandler(handler) reset_loggers() def test_storage_directory(self): self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'), 'objects/1/DEF/ABCDEF') def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) self.assert_('127.0.0.1' in myips) def test_whataremyips_error(self): def my_interfaces(): return ['eth0'] def my_ifaddress_error(interface): raise ValueError with nested( patch('netifaces.interfaces', my_interfaces), patch('netifaces.ifaddresses', my_ifaddress_error)): self.assertEquals(utils.whataremyips(), []) def test_whataremyips_ipv6(self): test_ipv6_address = '2001:6b0:dead:beef:2::32' test_interface = 'eth0' def my_ipv6_interfaces(): return ['eth0'] def my_ipv6_ifaddresses(interface): return {AF_INET6: [{'netmask': 'ffff:ffff:ffff:ffff::', 'addr': '%s%%%s' % (test_ipv6_address, test_interface)}]} with nested( patch('netifaces.interfaces', my_ipv6_interfaces), patch('netifaces.ifaddresses', my_ipv6_ifaddresses)): myips = utils.whataremyips() self.assertEquals(len(myips), 1) self.assertEquals(myips[0], test_ipv6_address) def test_hash_path(self): _prefix = utils.HASH_PATH_PREFIX utils.HASH_PATH_PREFIX = '' # Yes, these tests are deliberately very fragile. We want to make sure # that if someones changes the results hash_path produces, they know it try: self.assertEquals(utils.hash_path('a'), '1c84525acb02107ea475dcd3d09c2c58') self.assertEquals(utils.hash_path('a', 'c'), '33379ecb053aa5c9e356c68997cbb59e') self.assertEquals(utils.hash_path('a', 'c', 'o'), '06fbf0b514e5199dfc4e00f42eb5ea83') self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), '06fbf0b514e5199dfc4e00f42eb5ea83') self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=True), '\x06\xfb\xf0\xb5\x14\xe5\x19\x9d\xfcN' '\x00\xf4.\xb5\xea\x83') self.assertRaises(ValueError, utils.hash_path, 'a', object='o') utils.HASH_PATH_PREFIX = 'abcdef' self.assertEquals(utils.hash_path('a', 'c', 'o', raw_digest=False), '363f9b535bfb7d17a43a46a358afca0e') finally: utils.HASH_PATH_PREFIX = _prefix def test_load_libc_function(self): self.assert_(callable( utils.load_libc_function('printf'))) self.assert_(callable( utils.load_libc_function('some_not_real_function'))) def test_readconf(self): conf = '''[section1] foo = bar [section2] log_name = yarr''' # setup a real file fd, temppath = tempfile.mkstemp(dir='/tmp') with os.fdopen(fd, 'wb') as f: f.write(conf) make_filename = lambda: temppath # setup a file stream make_fp = lambda: StringIO(conf) for conf_object_maker in (make_filename, make_fp): conffile = conf_object_maker() result = utils.readconf(conffile) expected = {'__file__': conffile, 'log_name': None, 'section1': {'foo': 'bar'}, 'section2': {'log_name': 'yarr'}} self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1') expected = {'__file__': conffile, 'log_name': 'section1', 'foo': 'bar'} self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section2').get('log_name') expected = 'yarr' self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1', log_name='foo').get('log_name') expected = 'foo' self.assertEquals(result, expected) conffile = conf_object_maker() result = utils.readconf(conffile, 'section1', defaults={'bar': 'baz'}) expected = {'__file__': conffile, 'log_name': 'section1', 'foo': 'bar', 'bar': 'baz'} self.assertEquals(result, expected) self.assertRaises(SystemExit, utils.readconf, temppath, 'section3') os.unlink(temppath) self.assertRaises(SystemExit, utils.readconf, temppath) def test_readconf_raw(self): conf = '''[section1] foo = bar [section2] log_name = %(yarr)s''' # setup a real file fd, temppath = tempfile.mkstemp(dir='/tmp') with os.fdopen(fd, 'wb') as f: f.write(conf) make_filename = lambda: temppath # setup a file stream make_fp = lambda: StringIO(conf) for conf_object_maker in (make_filename, make_fp): conffile = conf_object_maker() result = utils.readconf(conffile, raw=True) expected = {'__file__': conffile, 'log_name': None, 'section1': {'foo': 'bar'}, 'section2': {'log_name': '%(yarr)s'}} self.assertEquals(result, expected) os.unlink(temppath) self.assertRaises(SystemExit, utils.readconf, temppath) def test_readconf_dir(self): config_dir = { 'server.conf.d/01.conf': """ [DEFAULT] port = 8080 foo = bar [section1] name=section1 """, 'server.conf.d/section2.conf': """ [DEFAULT] port = 8081 bar = baz [section2] name=section2 """, 'other-server.conf.d/01.conf': """ [DEFAULT] port = 8082 [section3] name=section3 """ } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as path: conf_dir = os.path.join(path, 'server.conf.d') conf = utils.readconf(conf_dir) expected = { '__file__': os.path.join(path, 'server.conf.d'), 'log_name': None, 'section1': { 'port': '8081', 'foo': 'bar', 'bar': 'baz', 'name': 'section1', }, 'section2': { 'port': '8081', 'foo': 'bar', 'bar': 'baz', 'name': 'section2', }, } self.assertEquals(conf, expected) def test_readconf_dir_ignores_hidden_and_nondotconf_files(self): config_dir = { 'server.conf.d/01.conf': """ [section1] port = 8080 """, 'server.conf.d/.01.conf.swp': """ [section] port = 8081 """, 'server.conf.d/01.conf-bak': """ [section] port = 8082 """, } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as path: conf_dir = os.path.join(path, 'server.conf.d') conf = utils.readconf(conf_dir) expected = { '__file__': os.path.join(path, 'server.conf.d'), 'log_name': None, 'section1': { 'port': '8080', }, } self.assertEquals(conf, expected) def test_drop_privileges(self): user = getuser() # over-ride os with mock required_func_calls = ('setgroups', 'setgid', 'setuid', 'setsid', 'chdir', 'umask') utils.os = MockOs(called_funcs=required_func_calls) # exercise the code utils.drop_privileges(user) for func in required_func_calls: self.assert_(utils.os.called_funcs[func]) import pwd self.assertEquals(pwd.getpwnam(user)[5], utils.os.environ['HOME']) groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] groups.append(pwd.getpwnam(user).pw_gid) self.assertEquals(set(groups), set(os.getgroups())) # reset; test same args, OSError trying to get session leader utils.os = MockOs(called_funcs=required_func_calls, raise_funcs=('setsid',)) for func in required_func_calls: self.assertFalse(utils.os.called_funcs.get(func, False)) utils.drop_privileges(user) for func in required_func_calls: self.assert_(utils.os.called_funcs[func]) def test_capture_stdio(self): # stubs logger = utils.get_logger(None, 'dummy') # mock utils system modules _orig_sys = utils.sys _orig_os = utils.os try: utils.sys = MockSys() utils.os = MockOs() # basic test utils.capture_stdio(logger) self.assert_(utils.sys.excepthook is not None) self.assertEquals(utils.os.closed_fds, utils.sys.stdio_fds) self.assert_(isinstance(utils.sys.stdout, utils.LoggerFileObject)) self.assert_(isinstance(utils.sys.stderr, utils.LoggerFileObject)) # reset; test same args, but exc when trying to close stdio utils.os = MockOs(raise_funcs=('dup2',)) utils.sys = MockSys() # test unable to close stdio utils.capture_stdio(logger) self.assert_(utils.sys.excepthook is not None) self.assertEquals(utils.os.closed_fds, []) self.assert_(isinstance(utils.sys.stdout, utils.LoggerFileObject)) self.assert_(isinstance(utils.sys.stderr, utils.LoggerFileObject)) # reset; test some other args utils.os = MockOs() utils.sys = MockSys() logger = utils.get_logger(None, log_to_console=True) # test console log utils.capture_stdio(logger, capture_stdout=False, capture_stderr=False) self.assert_(utils.sys.excepthook is not None) # when logging to console, stderr remains open self.assertEquals(utils.os.closed_fds, utils.sys.stdio_fds[:2]) reset_loggers() # stdio not captured self.assertFalse(isinstance(utils.sys.stdout, utils.LoggerFileObject)) self.assertFalse(isinstance(utils.sys.stderr, utils.LoggerFileObject)) reset_loggers() finally: utils.sys = _orig_sys utils.os = _orig_os def test_get_logger_console(self): reset_loggers() logger = utils.get_logger(None) console_handlers = [h for h in logger.logger.handlers if isinstance(h, logging.StreamHandler)] self.assertFalse(console_handlers) logger = utils.get_logger(None, log_to_console=True) console_handlers = [h for h in logger.logger.handlers if isinstance(h, logging.StreamHandler)] self.assert_(console_handlers) # make sure you can't have two console handlers self.assertEquals(len(console_handlers), 1) old_handler = console_handlers[0] logger = utils.get_logger(None, log_to_console=True) console_handlers = [h for h in logger.logger.handlers if isinstance(h, logging.StreamHandler)] self.assertEquals(len(console_handlers), 1) new_handler = console_handlers[0] self.assertNotEquals(new_handler, old_handler) reset_loggers() def verify_under_pseudo_time( self, func, target_runtime_ms=1, *args, **kwargs): curr_time = [42.0] def my_time(): curr_time[0] += 0.001 return curr_time[0] def my_sleep(duration): curr_time[0] += 0.001 curr_time[0] += duration with nested( patch('time.time', my_time), patch('time.sleep', my_sleep), patch('eventlet.sleep', my_sleep)): start = time.time() func(*args, **kwargs) # make sure it's accurate to 10th of a second, converting the time # difference to milliseconds, 100 milliseconds is 1/10 of a second diff_from_target_ms = abs( target_runtime_ms - ((time.time() - start) * 1000)) self.assertTrue(diff_from_target_ms < 100, "Expected %d < 100" % diff_from_target_ms) def test_ratelimit_sleep(self): def testfunc(): running_time = 0 for i in range(100): running_time = utils.ratelimit_sleep(running_time, -5) self.verify_under_pseudo_time(testfunc, target_runtime_ms=1) def testfunc(): running_time = 0 for i in range(100): running_time = utils.ratelimit_sleep(running_time, 0) self.verify_under_pseudo_time(testfunc, target_runtime_ms=1) def testfunc(): running_time = 0 for i in range(50): running_time = utils.ratelimit_sleep(running_time, 200) self.verify_under_pseudo_time(testfunc, target_runtime_ms=250) def test_ratelimit_sleep_with_incr(self): def testfunc(): running_time = 0 vals = [5, 17, 0, 3, 11, 30, 40, 4, 13, 2, -1] * 2 # adds up to 248 total = 0 for i in vals: running_time = utils.ratelimit_sleep(running_time, 500, incr_by=i) total += i self.assertEquals(248, total) self.verify_under_pseudo_time(testfunc, target_runtime_ms=500) def test_ratelimit_sleep_with_sleep(self): def testfunc(): running_time = 0 sleeps = [0] * 7 + [.2] * 3 + [0] * 30 for i in sleeps: running_time = utils.ratelimit_sleep(running_time, 40, rate_buffer=1) time.sleep(i) self.verify_under_pseudo_time(testfunc, target_runtime_ms=900) def test_urlparse(self): parsed = utils.urlparse('http://127.0.0.1/') self.assertEquals(parsed.scheme, 'http') self.assertEquals(parsed.hostname, '127.0.0.1') self.assertEquals(parsed.path, '/') parsed = utils.urlparse('http://127.0.0.1:8080/') self.assertEquals(parsed.port, 8080) parsed = utils.urlparse('https://127.0.0.1/') self.assertEquals(parsed.scheme, 'https') parsed = utils.urlparse('http://[::1]/') self.assertEquals(parsed.hostname, '::1') parsed = utils.urlparse('http://[::1]:8080/') self.assertEquals(parsed.hostname, '::1') self.assertEquals(parsed.port, 8080) parsed = utils.urlparse('www.example.com') self.assertEquals(parsed.hostname, '') def test_search_tree(self): # file match & ext miss with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: asdf = utils.search_tree(t, 'a*', '.conf') self.assertEquals(len(asdf), 1) self.assertEquals(asdf[0], os.path.join(t, 'asdf.conf')) # multi-file match & glob miss & sort with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t: app_bins = utils.search_tree(t, 'app*', 'bin') self.assertEquals(len(app_bins), 2) self.assertEquals(app_bins[0], os.path.join(t, 'apple.bin')) self.assertEquals(app_bins[1], os.path.join(t, 'application.bin')) # test file in folder & ext miss & glob miss files = ( 'sub/file1.ini', 'sub/file2.conf', 'sub.bin', 'bus.ini', 'bus/file3.ini', ) with temptree(files) as t: sub_ini = utils.search_tree(t, 'sub*', '.ini') self.assertEquals(len(sub_ini), 1) self.assertEquals(sub_ini[0], os.path.join(t, 'sub/file1.ini')) # test multi-file in folder & sub-folder & ext miss & glob miss files = ( 'folder_file.txt', 'folder/1.txt', 'folder/sub/2.txt', 'folder2/3.txt', 'Folder3/4.txt' 'folder.rc', ) with temptree(files) as t: folder_texts = utils.search_tree(t, 'folder*', '.txt') self.assertEquals(len(folder_texts), 4) f1 = os.path.join(t, 'folder_file.txt') f2 = os.path.join(t, 'folder/1.txt') f3 = os.path.join(t, 'folder/sub/2.txt') f4 = os.path.join(t, 'folder2/3.txt') for f in [f1, f2, f3, f4]: self.assert_(f in folder_texts) def test_search_tree_with_directory_ext_match(self): files = ( 'object-server/object-server.conf-base', 'object-server/1.conf.d/base.conf', 'object-server/1.conf.d/1.conf', 'object-server/2.conf.d/base.conf', 'object-server/2.conf.d/2.conf', 'object-server/3.conf.d/base.conf', 'object-server/3.conf.d/3.conf', 'object-server/4.conf.d/base.conf', 'object-server/4.conf.d/4.conf', ) with temptree(files) as t: conf_dirs = utils.search_tree(t, 'object-server', '.conf', dir_ext='conf.d') self.assertEquals(len(conf_dirs), 4) for i in range(4): conf_dir = os.path.join(t, 'object-server/%d.conf.d' % (i + 1)) self.assert_(conf_dir in conf_dirs) def test_write_file(self): with temptree([]) as t: file_name = os.path.join(t, 'test') utils.write_file(file_name, 'test') with open(file_name, 'r') as f: contents = f.read() self.assertEquals(contents, 'test') # and also subdirs file_name = os.path.join(t, 'subdir/test2') utils.write_file(file_name, 'test2') with open(file_name, 'r') as f: contents = f.read() self.assertEquals(contents, 'test2') # but can't over-write files file_name = os.path.join(t, 'subdir/test2/test3') self.assertRaises(IOError, utils.write_file, file_name, 'test3') def test_remove_file(self): with temptree([]) as t: file_name = os.path.join(t, 'blah.pid') # assert no raise self.assertEquals(os.path.exists(file_name), False) self.assertEquals(utils.remove_file(file_name), None) with open(file_name, 'w') as f: f.write('1') self.assert_(os.path.exists(file_name)) self.assertEquals(utils.remove_file(file_name), None) self.assertFalse(os.path.exists(file_name)) def test_human_readable(self): self.assertEquals(utils.human_readable(0), '0') self.assertEquals(utils.human_readable(1), '1') self.assertEquals(utils.human_readable(10), '10') self.assertEquals(utils.human_readable(100), '100') self.assertEquals(utils.human_readable(999), '999') self.assertEquals(utils.human_readable(1024), '1Ki') self.assertEquals(utils.human_readable(1535), '1Ki') self.assertEquals(utils.human_readable(1536), '2Ki') self.assertEquals(utils.human_readable(1047552), '1023Ki') self.assertEquals(utils.human_readable(1048063), '1023Ki') self.assertEquals(utils.human_readable(1048064), '1Mi') self.assertEquals(utils.human_readable(1048576), '1Mi') self.assertEquals(utils.human_readable(1073741824), '1Gi') self.assertEquals(utils.human_readable(1099511627776), '1Ti') self.assertEquals(utils.human_readable(1125899906842624), '1Pi') self.assertEquals(utils.human_readable(1152921504606846976), '1Ei') self.assertEquals(utils.human_readable(1180591620717411303424), '1Zi') self.assertEquals(utils.human_readable(1208925819614629174706176), '1Yi') self.assertEquals(utils.human_readable(1237940039285380274899124224), '1024Yi') def test_validate_sync_to(self): fname = 'container-sync-realms.conf' fcontents = ''' [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) for realms_conf in (None, csr): for goodurl, result in ( ('http://1.1.1.1/v1/a/c', (None, 'http://1.1.1.1/v1/a/c', None, None)), ('http://1.1.1.1:8080/a/c', (None, 'http://1.1.1.1:8080/a/c', None, None)), ('http://2.2.2.2/a/c', (None, 'http://2.2.2.2/a/c', None, None)), ('https://1.1.1.1/v1/a/c', (None, 'https://1.1.1.1/v1/a/c', None, None)), ('//US/DFW1/a/c', (None, 'http://dfw1.host/v1/a/c', 'US', '9ff3b71c849749dbaec4ccdd3cbab62b')), ('//us/DFW1/a/c', (None, 'http://dfw1.host/v1/a/c', 'US', '9ff3b71c849749dbaec4ccdd3cbab62b')), ('//us/dfw1/a/c', (None, 'http://dfw1.host/v1/a/c', 'US', '9ff3b71c849749dbaec4ccdd3cbab62b')), ('//', (None, None, None, None)), ('', (None, None, None, None))): if goodurl.startswith('//') and not realms_conf: self.assertEquals( utils.validate_sync_to( goodurl, ['1.1.1.1', '2.2.2.2'], realms_conf), (None, None, None, None)) else: self.assertEquals( utils.validate_sync_to( goodurl, ['1.1.1.1', '2.2.2.2'], realms_conf), result) for badurl, result in ( ('http://1.1.1.1', ('Path required in X-Container-Sync-To', None, None, None)), ('httpq://1.1.1.1/v1/a/c', ('Invalid scheme \'httpq\' in X-Container-Sync-To, ' 'must be "//", "http", or "https".', None, None, None)), ('http://1.1.1.1/v1/a/c?query', ('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To', None, None, None)), ('http://1.1.1.1/v1/a/c#frag', ('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To', None, None, None)), ('http://1.1.1.1/v1/a/c?query#frag', ('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To', None, None, None)), ('http://1.1.1.1/v1/a/c?query=param', ('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To', None, None, None)), ('http://1.1.1.1/v1/a/c?query=param#frag', ('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To', None, None, None)), ('http://1.1.1.2/v1/a/c', ("Invalid host '1.1.1.2' in X-Container-Sync-To", None, None, None)), ('//us/invalid/a/c', ("No cluster endpoint for 'us' 'invalid'", None, None, None)), ('//invalid/dfw1/a/c', ("No realm key for 'invalid'", None, None, None)), ('//us/invalid1/a/', ("Invalid X-Container-Sync-To format " "'//us/invalid1/a/'", None, None, None)), ('//us/invalid1/a', ("Invalid X-Container-Sync-To format " "'//us/invalid1/a'", None, None, None)), ('//us/invalid1/', ("Invalid X-Container-Sync-To format " "'//us/invalid1/'", None, None, None)), ('//us/invalid1', ("Invalid X-Container-Sync-To format " "'//us/invalid1'", None, None, None)), ('//us/', ("Invalid X-Container-Sync-To format " "'//us/'", None, None, None)), ('//us', ("Invalid X-Container-Sync-To format " "'//us'", None, None, None))): if badurl.startswith('//') and not realms_conf: self.assertEquals( utils.validate_sync_to( badurl, ['1.1.1.1', '2.2.2.2'], realms_conf), (None, None, None, None)) else: self.assertEquals( utils.validate_sync_to( badurl, ['1.1.1.1', '2.2.2.2'], realms_conf), result) def test_TRUE_VALUES(self): for v in utils.TRUE_VALUES: self.assertEquals(v, v.lower()) def test_config_true_value(self): orig_trues = utils.TRUE_VALUES try: utils.TRUE_VALUES = 'hello world'.split() for val in 'hello world HELLO WORLD'.split(): self.assertTrue(utils.config_true_value(val) is True) self.assertTrue(utils.config_true_value(True) is True) self.assertTrue(utils.config_true_value('foo') is False) self.assertTrue(utils.config_true_value(False) is False) finally: utils.TRUE_VALUES = orig_trues def test_config_auto_int_value(self): expectations = { # (value, default) : expected, ('1', 0): 1, (1, 0): 1, ('asdf', 0): ValueError, ('auto', 1): 1, ('AutO', 1): 1, ('Aut0', 1): ValueError, (None, 1): 1, } for (value, default), expected in expectations.items(): try: rv = utils.config_auto_int_value(value, default) except Exception as e: if e.__class__ is not expected: raise else: self.assertEquals(expected, rv) def test_streq_const_time(self): self.assertTrue(utils.streq_const_time('abc123', 'abc123')) self.assertFalse(utils.streq_const_time('a', 'aaaaa')) self.assertFalse(utils.streq_const_time('ABC123', 'abc123')) def test_quorum_size(self): expected_sizes = {1: 1, 2: 2, 3: 2, 4: 3, 5: 3} got_sizes = dict([(n, utils.quorum_size(n)) for n in expected_sizes]) self.assertEqual(expected_sizes, got_sizes) def test_rsync_ip_ipv4_localhost(self): self.assertEqual(utils.rsync_ip('127.0.0.1'), '127.0.0.1') def test_rsync_ip_ipv6_random_ip(self): self.assertEqual( utils.rsync_ip('fe80:0000:0000:0000:0202:b3ff:fe1e:8329'), '[fe80:0000:0000:0000:0202:b3ff:fe1e:8329]') def test_rsync_ip_ipv6_ipv4_compatible(self): self.assertEqual( utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]') def test_fallocate_reserve(self): class StatVFS(object): f_frsize = 1024 f_bavail = 1 def fstatvfs(fd): return StatVFS() orig_FALLOCATE_RESERVE = utils.FALLOCATE_RESERVE orig_fstatvfs = utils.os.fstatvfs try: fallocate = utils.FallocateWrapper(noop=True) utils.os.fstatvfs = fstatvfs # Want 1023 reserved, have 1024 * 1 free, so succeeds utils.FALLOCATE_RESERVE = 1023 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEquals(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) # Want 1023 reserved, have 512 * 2 free, so succeeds utils.FALLOCATE_RESERVE = 1023 StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 self.assertEquals(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) # Want 1024 reserved, have 1024 * 1 free, so fails utils.FALLOCATE_RESERVE = 1024 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1024 <= 1024') # Want 1024 reserved, have 512 * 2 free, so fails utils.FALLOCATE_RESERVE = 1024 StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1024 <= 1024') # Want 2048 reserved, have 1024 * 1 free, so fails utils.FALLOCATE_RESERVE = 2048 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1024 <= 2048') # Want 2048 reserved, have 512 * 2 free, so fails utils.FALLOCATE_RESERVE = 2048 StatVFS.f_frsize = 512 StatVFS.f_bavail = 2 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1024 <= 2048') # Want 1023 reserved, have 1024 * 1 free, but file size is 1, so # fails utils.FALLOCATE_RESERVE = 1023 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(1)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1023 <= 1023') # Want 1022 reserved, have 1024 * 1 free, and file size is 1, so # succeeds utils.FALLOCATE_RESERVE = 1022 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEquals(fallocate(0, 1, 0, ctypes.c_uint64(1)), 0) # Want 1023 reserved, have 1024 * 1 free, and file size is 0, so # succeeds utils.FALLOCATE_RESERVE = 1023 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 self.assertEquals(fallocate(0, 1, 0, ctypes.c_uint64(0)), 0) # Want 1024 reserved, have 1024 * 1 free, and even though # file size is 0, since we're under the reserve, fails utils.FALLOCATE_RESERVE = 1024 StatVFS.f_frsize = 1024 StatVFS.f_bavail = 1 exc = None try: fallocate(0, 1, 0, ctypes.c_uint64(0)) except OSError as err: exc = err self.assertEquals(str(exc), 'FALLOCATE_RESERVE fail 1024 <= 1024') finally: utils.FALLOCATE_RESERVE = orig_FALLOCATE_RESERVE utils.os.fstatvfs = orig_fstatvfs def test_fallocate_func(self): class FallocateWrapper(object): def __init__(self): self.last_call = None def __call__(self, *args): self.last_call = list(args) self.last_call[-1] = self.last_call[-1].value return 0 orig__sys_fallocate = utils._sys_fallocate try: utils._sys_fallocate = FallocateWrapper() # Ensure fallocate calls _sys_fallocate even with 0 bytes utils._sys_fallocate.last_call = None utils.fallocate(1234, 0) self.assertEquals(utils._sys_fallocate.last_call, [1234, 1, 0, 0]) # Ensure fallocate calls _sys_fallocate even with negative bytes utils._sys_fallocate.last_call = None utils.fallocate(1234, -5678) self.assertEquals(utils._sys_fallocate.last_call, [1234, 1, 0, 0]) # Ensure fallocate calls _sys_fallocate properly with positive # bytes utils._sys_fallocate.last_call = None utils.fallocate(1234, 1) self.assertEquals(utils._sys_fallocate.last_call, [1234, 1, 0, 1]) utils._sys_fallocate.last_call = None utils.fallocate(1234, 10 * 1024 * 1024 * 1024) self.assertEquals(utils._sys_fallocate.last_call, [1234, 1, 0, 10 * 1024 * 1024 * 1024]) finally: utils._sys_fallocate = orig__sys_fallocate def test_generate_trans_id(self): fake_time = 1366428370.5163341 with patch.object(utils.time, 'time', return_value=fake_time): trans_id = utils.generate_trans_id('') self.assertEquals(len(trans_id), 34) self.assertEquals(trans_id[:2], 'tx') self.assertEquals(trans_id[23], '-') self.assertEquals(int(trans_id[24:], 16), int(fake_time)) with patch.object(utils.time, 'time', return_value=fake_time): trans_id = utils.generate_trans_id('-suffix') self.assertEquals(len(trans_id), 41) self.assertEquals(trans_id[:2], 'tx') self.assertEquals(trans_id[34:], '-suffix') self.assertEquals(trans_id[23], '-') self.assertEquals(int(trans_id[24:34], 16), int(fake_time)) def test_get_trans_id_time(self): ts = utils.get_trans_id_time('tx8c8bc884cdaf499bb29429aa9c46946e') self.assertEquals(ts, None) ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-0051720c06') self.assertEquals(ts, 1366428678) self.assertEquals( time.asctime(time.gmtime(ts)) + ' UTC', 'Sat Apr 20 03:31:18 2013 UTC') ts = utils.get_trans_id_time( 'tx1df4ff4f55ea45f7b2ec2-0051720c06-suffix') self.assertEquals(ts, 1366428678) self.assertEquals( time.asctime(time.gmtime(ts)) + ' UTC', 'Sat Apr 20 03:31:18 2013 UTC') ts = utils.get_trans_id_time('') self.assertEquals(ts, None) ts = utils.get_trans_id_time('garbage') self.assertEquals(ts, None) ts = utils.get_trans_id_time('tx1df4ff4f55ea45f7b2ec2-almostright') self.assertEquals(ts, None) def test_tpool_reraise(self): with patch.object(utils.tpool, 'execute', lambda f: f()): self.assertTrue( utils.tpool_reraise(MagicMock(return_value='test1')), 'test1') self.assertRaises( Exception, utils.tpool_reraise, MagicMock(side_effect=Exception('test2'))) self.assertRaises( BaseException, utils.tpool_reraise, MagicMock(side_effect=BaseException('test3'))) def test_lock_file(self): flags = os.O_CREAT | os.O_RDWR with NamedTemporaryFile(delete=False) as nt: nt.write("test string") nt.flush() nt.close() with utils.lock_file(nt.name, unlink=False) as f: self.assertEqual(f.read(), "test string") # we have a lock, now let's try to get a newer one fd = os.open(nt.name, flags) self.assertRaises(IOError, fcntl.flock, fd, fcntl.LOCK_EX | fcntl.LOCK_NB) with utils.lock_file(nt.name, unlink=False, append=True) as f: self.assertEqual(f.read(), "test string") f.seek(0) f.write("\nanother string") f.flush() f.seek(0) self.assertEqual(f.read(), "test string\nanother string") # we have a lock, now let's try to get a newer one fd = os.open(nt.name, flags) self.assertRaises(IOError, fcntl.flock, fd, fcntl.LOCK_EX | fcntl.LOCK_NB) with utils.lock_file(nt.name, timeout=3, unlink=False) as f: try: with utils.lock_file( nt.name, timeout=1, unlink=False) as f: self.assertTrue( False, "Expected LockTimeout exception") except LockTimeout: pass with utils.lock_file(nt.name, unlink=True) as f: self.assertEqual(f.read(), "test string\nanother string") # we have a lock, now let's try to get a newer one fd = os.open(nt.name, flags) self.assertRaises( IOError, fcntl.flock, fd, fcntl.LOCK_EX | fcntl.LOCK_NB) self.assertRaises(OSError, os.remove, nt.name) def test_ismount_path_does_not_exist(self): tmpdir = mkdtemp() try: self.assertFalse(utils.ismount(os.path.join(tmpdir, 'bar'))) finally: shutil.rmtree(tmpdir) def test_ismount_path_not_mount(self): tmpdir = mkdtemp() try: self.assertFalse(utils.ismount(tmpdir)) finally: shutil.rmtree(tmpdir) def test_ismount_path_error(self): def _mock_os_lstat(path): raise OSError(13, "foo") tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): # Raises exception with _raw -- see next test. utils.ismount(tmpdir) finally: shutil.rmtree(tmpdir) def test_ismount_raw_path_error(self): def _mock_os_lstat(path): raise OSError(13, "foo") tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): self.assertRaises(OSError, utils.ismount_raw, tmpdir) finally: shutil.rmtree(tmpdir) def test_ismount_path_is_symlink(self): tmpdir = mkdtemp() try: link = os.path.join(tmpdir, "tmp") os.symlink("/tmp", link) self.assertFalse(utils.ismount(link)) finally: shutil.rmtree(tmpdir) def test_ismount_path_is_root(self): self.assertTrue(utils.ismount('/')) def test_ismount_parent_path_error(self): _os_lstat = os.lstat def _mock_os_lstat(path): if path.endswith(".."): raise OSError(13, "foo") else: return _os_lstat(path) tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): # Raises exception with _raw -- see next test. utils.ismount(tmpdir) finally: shutil.rmtree(tmpdir) def test_ismount_raw_parent_path_error(self): _os_lstat = os.lstat def _mock_os_lstat(path): if path.endswith(".."): raise OSError(13, "foo") else: return _os_lstat(path) tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): self.assertRaises(OSError, utils.ismount_raw, tmpdir) finally: shutil.rmtree(tmpdir) def test_ismount_successes_dev(self): _os_lstat = os.lstat class MockStat(object): def __init__(self, mode, dev, ino): self.st_mode = mode self.st_dev = dev self.st_ino = ino def _mock_os_lstat(path): if path.endswith(".."): parent = _os_lstat(path) return MockStat(parent.st_mode, parent.st_dev + 1, parent.st_ino) else: return _os_lstat(path) tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): self.assertTrue(utils.ismount(tmpdir)) finally: shutil.rmtree(tmpdir) def test_ismount_successes_ino(self): _os_lstat = os.lstat class MockStat(object): def __init__(self, mode, dev, ino): self.st_mode = mode self.st_dev = dev self.st_ino = ino def _mock_os_lstat(path): if path.endswith(".."): return _os_lstat(path) else: parent_path = os.path.join(path, "..") child = _os_lstat(path) parent = _os_lstat(parent_path) return MockStat(child.st_mode, parent.st_ino, child.st_dev) tmpdir = mkdtemp() try: with patch("os.lstat", _mock_os_lstat): self.assertTrue(utils.ismount(tmpdir)) finally: shutil.rmtree(tmpdir) def test_parse_content_type(self): self.assertEquals(utils.parse_content_type('text/plain'), ('text/plain', [])) self.assertEquals(utils.parse_content_type('text/plain;charset=utf-8'), ('text/plain', [('charset', 'utf-8')])) self.assertEquals( utils.parse_content_type('text/plain;hello="world";charset=utf-8'), ('text/plain', [('hello', '"world"'), ('charset', 'utf-8')])) self.assertEquals( utils.parse_content_type('text/plain; hello="world"; a=b'), ('text/plain', [('hello', '"world"'), ('a', 'b')])) self.assertEquals( utils.parse_content_type(r'text/plain; x="\""; a=b'), ('text/plain', [('x', r'"\""'), ('a', 'b')])) self.assertEquals( utils.parse_content_type(r'text/plain; x; a=b'), ('text/plain', [('x', ''), ('a', 'b')])) self.assertEquals( utils.parse_content_type(r'text/plain; x="\""; a'), ('text/plain', [('x', r'"\""'), ('a', '')])) def test_override_bytes_from_content_type(self): listing_dict = { 'bytes': 1234, 'hash': 'asdf', 'name': 'zxcv', 'content_type': 'text/plain; hello="world"; swift_bytes=15'} utils.override_bytes_from_content_type(listing_dict, logger=FakeLogger()) self.assertEquals(listing_dict['bytes'], 15) self.assertEquals(listing_dict['content_type'], 'text/plain;hello="world"') listing_dict = { 'bytes': 1234, 'hash': 'asdf', 'name': 'zxcv', 'content_type': 'text/plain; hello="world"; swift_bytes=hey'} utils.override_bytes_from_content_type(listing_dict, logger=FakeLogger()) self.assertEquals(listing_dict['bytes'], 1234) self.assertEquals(listing_dict['content_type'], 'text/plain;hello="world"') def test_quote(self): res = utils.quote('/v1/a/c3/subdirx/') assert res == '/v1/a/c3/subdirx/' res = utils.quote('/v1/a&b/c3/subdirx/') assert res == '/v1/a%26b/c3/subdirx/' res = utils.quote('/v1/a&b/c3/subdirx/', safe='&') assert res == '%2Fv1%2Fa&b%2Fc3%2Fsubdirx%2F' unicode_sample = u'\uc77c\uc601' account = 'abc_' + unicode_sample valid_utf8_str = utils.get_valid_utf8_str(account) account = 'abc_' + unicode_sample.encode('utf-8')[::-1] invalid_utf8_str = utils.get_valid_utf8_str(account) self.assertEquals('abc_%EC%9D%BC%EC%98%81', utils.quote(valid_utf8_str)) self.assertEquals('abc_%EF%BF%BD%EF%BF%BD%EC%BC%9D%EF%BF%BD', utils.quote(invalid_utf8_str)) def test_get_hmac(self): self.assertEquals( utils.get_hmac('GET', '/path', 1, 'abc'), 'b17f6ff8da0e251737aa9e3ee69a881e3e092e2f') class TestSwiftInfo(unittest.TestCase): def tearDown(self): utils._swift_info = {} utils._swift_admin_info = {} def test_register_swift_info(self): utils.register_swift_info(foo='bar') utils.register_swift_info(lorem='ipsum') utils.register_swift_info('cap1', cap1_foo='cap1_bar') utils.register_swift_info('cap1', cap1_lorem='cap1_ipsum') self.assertTrue('swift' in utils._swift_info) self.assertTrue('foo' in utils._swift_info['swift']) self.assertEqual(utils._swift_info['swift']['foo'], 'bar') self.assertTrue('lorem' in utils._swift_info['swift']) self.assertEqual(utils._swift_info['swift']['lorem'], 'ipsum') self.assertTrue('cap1' in utils._swift_info) self.assertTrue('cap1_foo' in utils._swift_info['cap1']) self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') self.assertTrue('cap1_lorem' in utils._swift_info['cap1']) self.assertEqual(utils._swift_info['cap1']['cap1_lorem'], 'cap1_ipsum') self.assertRaises(ValueError, utils.register_swift_info, 'admin', foo='bar') self.assertRaises(ValueError, utils.register_swift_info, 'disallowed_sections', disallowed_sections=None) def test_get_swift_info(self): utils._swift_info = {'swift': {'foo': 'bar'}, 'cap1': {'cap1_foo': 'cap1_bar'}} utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} info = utils.get_swift_info() self.assertTrue('admin' not in info) self.assertTrue('swift' in info) self.assertTrue('foo' in info['swift']) self.assertEqual(utils._swift_info['swift']['foo'], 'bar') self.assertTrue('cap1' in info) self.assertTrue('cap1_foo' in info['cap1']) self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') def test_get_swift_info_with_disallowed_sections(self): utils._swift_info = {'swift': {'foo': 'bar'}, 'cap1': {'cap1_foo': 'cap1_bar'}, 'cap2': {'cap2_foo': 'cap2_bar'}, 'cap3': {'cap3_foo': 'cap3_bar'}} utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} info = utils.get_swift_info(disallowed_sections=['cap1', 'cap3']) self.assertTrue('admin' not in info) self.assertTrue('swift' in info) self.assertTrue('foo' in info['swift']) self.assertEqual(info['swift']['foo'], 'bar') self.assertTrue('cap1' not in info) self.assertTrue('cap2' in info) self.assertTrue('cap2_foo' in info['cap2']) self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') self.assertTrue('cap3' not in info) def test_register_swift_admin_info(self): utils.register_swift_info(admin=True, admin_foo='admin_bar') utils.register_swift_info(admin=True, admin_lorem='admin_ipsum') utils.register_swift_info('cap1', admin=True, ac1_foo='ac1_bar') utils.register_swift_info('cap1', admin=True, ac1_lorem='ac1_ipsum') self.assertTrue('swift' in utils._swift_admin_info) self.assertTrue('admin_foo' in utils._swift_admin_info['swift']) self.assertEqual( utils._swift_admin_info['swift']['admin_foo'], 'admin_bar') self.assertTrue('admin_lorem' in utils._swift_admin_info['swift']) self.assertEqual( utils._swift_admin_info['swift']['admin_lorem'], 'admin_ipsum') self.assertTrue('cap1' in utils._swift_admin_info) self.assertTrue('ac1_foo' in utils._swift_admin_info['cap1']) self.assertEqual( utils._swift_admin_info['cap1']['ac1_foo'], 'ac1_bar') self.assertTrue('ac1_lorem' in utils._swift_admin_info['cap1']) self.assertEqual( utils._swift_admin_info['cap1']['ac1_lorem'], 'ac1_ipsum') self.assertTrue('swift' not in utils._swift_info) self.assertTrue('cap1' not in utils._swift_info) def test_get_swift_admin_info(self): utils._swift_info = {'swift': {'foo': 'bar'}, 'cap1': {'cap1_foo': 'cap1_bar'}} utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} info = utils.get_swift_info(admin=True) self.assertTrue('admin' in info) self.assertTrue('admin_cap1' in info['admin']) self.assertTrue('ac1_foo' in info['admin']['admin_cap1']) self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') self.assertTrue('swift' in info) self.assertTrue('foo' in info['swift']) self.assertEqual(utils._swift_info['swift']['foo'], 'bar') self.assertTrue('cap1' in info) self.assertTrue('cap1_foo' in info['cap1']) self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') def test_get_swift_admin_info_with_disallowed_sections(self): utils._swift_info = {'swift': {'foo': 'bar'}, 'cap1': {'cap1_foo': 'cap1_bar'}, 'cap2': {'cap2_foo': 'cap2_bar'}, 'cap3': {'cap3_foo': 'cap3_bar'}} utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} info = utils.get_swift_info( admin=True, disallowed_sections=['cap1', 'cap3']) self.assertTrue('admin' in info) self.assertTrue('admin_cap1' in info['admin']) self.assertTrue('ac1_foo' in info['admin']['admin_cap1']) self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') self.assertTrue('disallowed_sections' in info['admin']) self.assertTrue('cap1' in info['admin']['disallowed_sections']) self.assertTrue('cap2' not in info['admin']['disallowed_sections']) self.assertTrue('cap3' in info['admin']['disallowed_sections']) self.assertTrue('swift' in info) self.assertTrue('foo' in info['swift']) self.assertEqual(info['swift']['foo'], 'bar') self.assertTrue('cap1' not in info) self.assertTrue('cap2' in info) self.assertTrue('cap2_foo' in info['cap2']) self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') self.assertTrue('cap3' not in info) class TestFileLikeIter(unittest.TestCase): def test_iter_file_iter(self): in_iter = ['abc', 'de', 'fghijk', 'l'] chunks = [] for chunk in utils.FileLikeIter(in_iter): chunks.append(chunk) self.assertEquals(chunks, in_iter) def test_next(self): in_iter = ['abc', 'de', 'fghijk', 'l'] chunks = [] iter_file = utils.FileLikeIter(in_iter) while True: try: chunk = iter_file.next() except StopIteration: break chunks.append(chunk) self.assertEquals(chunks, in_iter) def test_read(self): in_iter = ['abc', 'de', 'fghijk', 'l'] iter_file = utils.FileLikeIter(in_iter) self.assertEquals(iter_file.read(), ''.join(in_iter)) def test_read_with_size(self): in_iter = ['abc', 'de', 'fghijk', 'l'] chunks = [] iter_file = utils.FileLikeIter(in_iter) while True: chunk = iter_file.read(2) if not chunk: break self.assertTrue(len(chunk) <= 2) chunks.append(chunk) self.assertEquals(''.join(chunks), ''.join(in_iter)) def test_read_with_size_zero(self): # makes little sense, but file supports it, so... self.assertEquals(utils.FileLikeIter('abc').read(0), '') def test_readline(self): in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] lines = [] iter_file = utils.FileLikeIter(in_iter) while True: line = iter_file.readline() if not line: break lines.append(line) self.assertEquals( lines, [v if v == 'trailing.' else v + '\n' for v in ''.join(in_iter).split('\n')]) def test_readline2(self): self.assertEquals( utils.FileLikeIter(['abc', 'def\n']).readline(4), 'abcd') def test_readline3(self): self.assertEquals( utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(), ('a' * 1111) + 'bc\n') def test_readline_with_size(self): in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] lines = [] iter_file = utils.FileLikeIter(in_iter) while True: line = iter_file.readline(2) if not line: break lines.append(line) self.assertEquals( lines, ['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n', 'tr', 'ai', 'li', 'ng', '.']) def test_readlines(self): in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] lines = utils.FileLikeIter(in_iter).readlines() self.assertEquals( lines, [v if v == 'trailing.' else v + '\n' for v in ''.join(in_iter).split('\n')]) def test_readlines_with_size(self): in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.'] iter_file = utils.FileLikeIter(in_iter) lists_of_lines = [] while True: lines = iter_file.readlines(2) if not lines: break lists_of_lines.append(lines) self.assertEquals( lists_of_lines, [['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'], ['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']]) def test_close(self): iter_file = utils.FileLikeIter('abcdef') self.assertEquals(iter_file.next(), 'a') iter_file.close() self.assertTrue(iter_file.closed) self.assertRaises(ValueError, iter_file.next) self.assertRaises(ValueError, iter_file.read) self.assertRaises(ValueError, iter_file.readline) self.assertRaises(ValueError, iter_file.readlines) # Just make sure repeated close calls don't raise an Exception iter_file.close() self.assertTrue(iter_file.closed) class TestStatsdLogging(unittest.TestCase): def test_get_logger_statsd_client_not_specified(self): logger = utils.get_logger({}, 'some-name', log_route='some-route') # white-box construction validation self.assertEqual(None, logger.logger.statsd_client) def test_get_logger_statsd_client_defaults(self): logger = utils.get_logger({'log_statsd_host': 'some.host.com'}, 'some-name', log_route='some-route') # white-box construction validation self.assert_(isinstance(logger.logger.statsd_client, utils.StatsdClient)) self.assertEqual(logger.logger.statsd_client._host, 'some.host.com') self.assertEqual(logger.logger.statsd_client._port, 8125) self.assertEqual(logger.logger.statsd_client._prefix, 'some-name.') self.assertEqual(logger.logger.statsd_client._default_sample_rate, 1) logger.set_statsd_prefix('some-name.more-specific') self.assertEqual(logger.logger.statsd_client._prefix, 'some-name.more-specific.') logger.set_statsd_prefix('') self.assertEqual(logger.logger.statsd_client._prefix, '') def test_get_logger_statsd_client_non_defaults(self): logger = utils.get_logger({ 'log_statsd_host': 'another.host.com', 'log_statsd_port': '9876', 'log_statsd_default_sample_rate': '0.75', 'log_statsd_sample_rate_factor': '0.81', 'log_statsd_metric_prefix': 'tomato.sauce', }, 'some-name', log_route='some-route') self.assertEqual(logger.logger.statsd_client._prefix, 'tomato.sauce.some-name.') logger.set_statsd_prefix('some-name.more-specific') self.assertEqual(logger.logger.statsd_client._prefix, 'tomato.sauce.some-name.more-specific.') logger.set_statsd_prefix('') self.assertEqual(logger.logger.statsd_client._prefix, 'tomato.sauce.') self.assertEqual(logger.logger.statsd_client._host, 'another.host.com') self.assertEqual(logger.logger.statsd_client._port, 9876) self.assertEqual(logger.logger.statsd_client._default_sample_rate, 0.75) self.assertEqual(logger.logger.statsd_client._sample_rate_factor, 0.81) def test_sample_rates(self): logger = utils.get_logger({'log_statsd_host': 'some.host.com'}) mock_socket = MockUdpSocket() # encapsulation? what's that? statsd_client = logger.logger.statsd_client self.assertTrue(statsd_client.random is random.random) statsd_client._open_socket = lambda *_: mock_socket statsd_client.random = lambda: 0.50001 logger.increment('tribbles', sample_rate=0.5) self.assertEqual(len(mock_socket.sent), 0) statsd_client.random = lambda: 0.49999 logger.increment('tribbles', sample_rate=0.5) self.assertEqual(len(mock_socket.sent), 1) payload = mock_socket.sent[0][0] self.assertTrue(payload.endswith("|@0.5")) def test_sample_rates_with_sample_rate_factor(self): logger = utils.get_logger({ 'log_statsd_host': 'some.host.com', 'log_statsd_default_sample_rate': '0.82', 'log_statsd_sample_rate_factor': '0.91', }) effective_sample_rate = 0.82 * 0.91 mock_socket = MockUdpSocket() # encapsulation? what's that? statsd_client = logger.logger.statsd_client self.assertTrue(statsd_client.random is random.random) statsd_client._open_socket = lambda *_: mock_socket statsd_client.random = lambda: effective_sample_rate + 0.001 logger.increment('tribbles') self.assertEqual(len(mock_socket.sent), 0) statsd_client.random = lambda: effective_sample_rate - 0.001 logger.increment('tribbles') self.assertEqual(len(mock_socket.sent), 1) payload = mock_socket.sent[0][0] self.assertTrue(payload.endswith("|@%s" % effective_sample_rate), payload) effective_sample_rate = 0.587 * 0.91 statsd_client.random = lambda: effective_sample_rate - 0.001 logger.increment('tribbles', sample_rate=0.587) self.assertEqual(len(mock_socket.sent), 2) payload = mock_socket.sent[1][0] self.assertTrue(payload.endswith("|@%s" % effective_sample_rate), payload) def test_timing_stats(self): class MockController(object): def __init__(self, status): self.status = status self.logger = self self.args = () self.called = 'UNKNOWN' def timing_since(self, *args): self.called = 'timing' self.args = args @utils.timing_stats() def METHOD(controller): return Response(status=controller.status) mock_controller = MockController(200) METHOD(mock_controller) self.assertEquals(mock_controller.called, 'timing') self.assertEquals(len(mock_controller.args), 2) self.assertEquals(mock_controller.args[0], 'METHOD.timing') self.assert_(mock_controller.args[1] > 0) mock_controller = MockController(404) METHOD(mock_controller) self.assertEquals(len(mock_controller.args), 2) self.assertEquals(mock_controller.called, 'timing') self.assertEquals(mock_controller.args[0], 'METHOD.timing') self.assert_(mock_controller.args[1] > 0) mock_controller = MockController(401) METHOD(mock_controller) self.assertEquals(len(mock_controller.args), 2) self.assertEquals(mock_controller.called, 'timing') self.assertEquals(mock_controller.args[0], 'METHOD.errors.timing') self.assert_(mock_controller.args[1] > 0) class UnsafeXrange(object): """ Like xrange(limit), but with extra context switching to screw things up. """ def __init__(self, upper_bound): self.current = 0 self.concurrent_calls = 0 self.upper_bound = upper_bound self.concurrent_call = False def __iter__(self): return self def next(self): if self.concurrent_calls > 0: self.concurrent_call = True self.concurrent_calls += 1 try: if self.current >= self.upper_bound: raise StopIteration else: val = self.current self.current += 1 eventlet.sleep() # yield control return val finally: self.concurrent_calls -= 1 class TestAffinityKeyFunction(unittest.TestCase): def setUp(self): self.nodes = [dict(id=0, region=1, zone=1), dict(id=1, region=1, zone=2), dict(id=2, region=2, zone=1), dict(id=3, region=2, zone=2), dict(id=4, region=3, zone=1), dict(id=5, region=3, zone=2), dict(id=6, region=4, zone=0), dict(id=7, region=4, zone=1)] def test_single_region(self): keyfn = utils.affinity_key_function("r3=1") ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([4, 5, 0, 1, 2, 3, 6, 7], ids) def test_bogus_value(self): self.assertRaises(ValueError, utils.affinity_key_function, "r3") self.assertRaises(ValueError, utils.affinity_key_function, "r3=elephant") def test_empty_value(self): # Empty's okay, it just means no preference keyfn = utils.affinity_key_function("") self.assert_(callable(keyfn)) ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([0, 1, 2, 3, 4, 5, 6, 7], ids) def test_all_whitespace_value(self): # Empty's okay, it just means no preference keyfn = utils.affinity_key_function(" \n") self.assert_(callable(keyfn)) ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([0, 1, 2, 3, 4, 5, 6, 7], ids) def test_with_zone_zero(self): keyfn = utils.affinity_key_function("r4z0=1") ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([6, 0, 1, 2, 3, 4, 5, 7], ids) def test_multiple(self): keyfn = utils.affinity_key_function("r1=100, r4=200, r3z1=1") ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([4, 0, 1, 6, 7, 2, 3, 5], ids) def test_more_specific_after_less_specific(self): keyfn = utils.affinity_key_function("r2=100, r2z2=50") ids = [n['id'] for n in sorted(self.nodes, key=keyfn)] self.assertEqual([3, 2, 0, 1, 4, 5, 6, 7], ids) class TestAffinityLocalityPredicate(unittest.TestCase): def setUp(self): self.nodes = [dict(id=0, region=1, zone=1), dict(id=1, region=1, zone=2), dict(id=2, region=2, zone=1), dict(id=3, region=2, zone=2), dict(id=4, region=3, zone=1), dict(id=5, region=3, zone=2), dict(id=6, region=4, zone=0), dict(id=7, region=4, zone=1)] def test_empty(self): pred = utils.affinity_locality_predicate('') self.assert_(pred is None) def test_region(self): pred = utils.affinity_locality_predicate('r1') self.assert_(callable(pred)) ids = [n['id'] for n in self.nodes if pred(n)] self.assertEqual([0, 1], ids) def test_zone(self): pred = utils.affinity_locality_predicate('r1z1') self.assert_(callable(pred)) ids = [n['id'] for n in self.nodes if pred(n)] self.assertEqual([0], ids) def test_multiple(self): pred = utils.affinity_locality_predicate('r1, r3, r4z0') self.assert_(callable(pred)) ids = [n['id'] for n in self.nodes if pred(n)] self.assertEqual([0, 1, 4, 5, 6], ids) def test_invalid(self): self.assertRaises(ValueError, utils.affinity_locality_predicate, 'falafel') self.assertRaises(ValueError, utils.affinity_locality_predicate, 'r8zQ') self.assertRaises(ValueError, utils.affinity_locality_predicate, 'r2d2') self.assertRaises(ValueError, utils.affinity_locality_predicate, 'r1z1=1') class TestRateLimitedIterator(unittest.TestCase): def run_under_pseudo_time( self, func, *args, **kwargs): curr_time = [42.0] def my_time(): curr_time[0] += 0.001 return curr_time[0] def my_sleep(duration): curr_time[0] += 0.001 curr_time[0] += duration with nested( patch('time.time', my_time), patch('eventlet.sleep', my_sleep)): return func(*args, **kwargs) def test_rate_limiting(self): def testfunc(): limited_iterator = utils.RateLimitedIterator(xrange(9999), 100) got = [] started_at = time.time() try: while time.time() - started_at < 0.1: got.append(limited_iterator.next()) except StopIteration: pass return got got = self.run_under_pseudo_time(testfunc) # it's 11, not 10, because ratelimiting doesn't apply to the very # first element. self.assertEquals(len(got), 11) def test_limit_after(self): def testfunc(): limited_iterator = utils.RateLimitedIterator( xrange(9999), 100, limit_after=5) got = [] started_at = time.time() try: while time.time() - started_at < 0.1: got.append(limited_iterator.next()) except StopIteration: pass return got got = self.run_under_pseudo_time(testfunc) # it's 16, not 15, because ratelimiting doesn't apply to the very # first element. self.assertEquals(len(got), 16) class TestGreenthreadSafeIterator(unittest.TestCase): def increment(self, iterable): plus_ones = [] for n in iterable: plus_ones.append(n + 1) return plus_ones def test_setup_works(self): # it should work without concurrent access self.assertEquals([0, 1, 2, 3], list(UnsafeXrange(4))) iterable = UnsafeXrange(10) pile = eventlet.GreenPile(2) for _ in xrange(2): pile.spawn(self.increment, iterable) sorted([resp for resp in pile]) self.assertTrue( iterable.concurrent_call, 'test setup is insufficiently crazy') def test_access_is_serialized(self): pile = eventlet.GreenPile(2) unsafe_iterable = UnsafeXrange(10) iterable = utils.GreenthreadSafeIterator(unsafe_iterable) for _ in xrange(2): pile.spawn(self.increment, iterable) response = sorted(sum([resp for resp in pile], [])) self.assertEquals(range(1, 11), response) self.assertTrue( not unsafe_iterable.concurrent_call, 'concurrent call occurred') class TestStatsdLoggingDelegation(unittest.TestCase): def setUp(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.bind(('localhost', 0)) self.port = self.sock.getsockname()[1] self.queue = Queue() self.reader_thread = threading.Thread(target=self.statsd_reader) self.reader_thread.setDaemon(1) self.reader_thread.start() def tearDown(self): # The "no-op when disabled" test doesn't set up a real logger, so # create one here so we can tell the reader thread to stop. if not getattr(self, 'logger', None): self.logger = utils.get_logger({ 'log_statsd_host': 'localhost', 'log_statsd_port': str(self.port), }, 'some-name') self.logger.increment('STOP') self.reader_thread.join(timeout=4) self.sock.close() del self.logger def statsd_reader(self): while True: try: payload = self.sock.recv(4096) if payload and 'STOP' in payload: return 42 self.queue.put(payload) except Exception as e: sys.stderr.write('statsd_reader thread: %r' % (e,)) break def _send_and_get(self, sender_fn, *args, **kwargs): """ Because the client library may not actually send a packet with sample_rate < 1, we keep trying until we get one through. """ got = None while not got: sender_fn(*args, **kwargs) try: got = self.queue.get(timeout=0.5) except Empty: pass return got def assertStat(self, expected, sender_fn, *args, **kwargs): got = self._send_and_get(sender_fn, *args, **kwargs) return self.assertEqual(expected, got) def assertStatMatches(self, expected_regexp, sender_fn, *args, **kwargs): got = self._send_and_get(sender_fn, *args, **kwargs) return self.assert_(re.search(expected_regexp, got), [got, expected_regexp]) def test_methods_are_no_ops_when_not_enabled(self): logger = utils.get_logger({ # No "log_statsd_host" means "disabled" 'log_statsd_port': str(self.port), }, 'some-name') # Delegate methods are no-ops self.assertEqual(None, logger.update_stats('foo', 88)) self.assertEqual(None, logger.update_stats('foo', 88, 0.57)) self.assertEqual(None, logger.update_stats('foo', 88, sample_rate=0.61)) self.assertEqual(None, logger.increment('foo')) self.assertEqual(None, logger.increment('foo', 0.57)) self.assertEqual(None, logger.increment('foo', sample_rate=0.61)) self.assertEqual(None, logger.decrement('foo')) self.assertEqual(None, logger.decrement('foo', 0.57)) self.assertEqual(None, logger.decrement('foo', sample_rate=0.61)) self.assertEqual(None, logger.timing('foo', 88.048)) self.assertEqual(None, logger.timing('foo', 88.57, 0.34)) self.assertEqual(None, logger.timing('foo', 88.998, sample_rate=0.82)) self.assertEqual(None, logger.timing_since('foo', 8938)) self.assertEqual(None, logger.timing_since('foo', 8948, 0.57)) self.assertEqual(None, logger.timing_since('foo', 849398, sample_rate=0.61)) # Now, the queue should be empty (no UDP packets sent) self.assertRaises(Empty, self.queue.get_nowait) def test_delegate_methods_with_no_default_sample_rate(self): self.logger = utils.get_logger({ 'log_statsd_host': 'localhost', 'log_statsd_port': str(self.port), }, 'some-name') self.assertStat('some-name.some.counter:1|c', self.logger.increment, 'some.counter') self.assertStat('some-name.some.counter:-1|c', self.logger.decrement, 'some.counter') self.assertStat('some-name.some.operation:4900.0|ms', self.logger.timing, 'some.operation', 4.9 * 1000) self.assertStatMatches('some-name\.another\.operation:\d+\.\d+\|ms', self.logger.timing_since, 'another.operation', time.time()) self.assertStat('some-name.another.counter:42|c', self.logger.update_stats, 'another.counter', 42) # Each call can override the sample_rate (also, bonus prefix test) self.logger.set_statsd_prefix('pfx') self.assertStat('pfx.some.counter:1|c|@0.972', self.logger.increment, 'some.counter', sample_rate=0.972) self.assertStat('pfx.some.counter:-1|c|@0.972', self.logger.decrement, 'some.counter', sample_rate=0.972) self.assertStat('pfx.some.operation:4900.0|ms|@0.972', self.logger.timing, 'some.operation', 4.9 * 1000, sample_rate=0.972) self.assertStatMatches('pfx\.another\.op:\d+\.\d+\|ms|@0.972', self.logger.timing_since, 'another.op', time.time(), sample_rate=0.972) self.assertStat('pfx.another.counter:3|c|@0.972', self.logger.update_stats, 'another.counter', 3, sample_rate=0.972) # Can override sample_rate with non-keyword arg self.logger.set_statsd_prefix('') self.assertStat('some.counter:1|c|@0.939', self.logger.increment, 'some.counter', 0.939) self.assertStat('some.counter:-1|c|@0.939', self.logger.decrement, 'some.counter', 0.939) self.assertStat('some.operation:4900.0|ms|@0.939', self.logger.timing, 'some.operation', 4.9 * 1000, 0.939) self.assertStatMatches('another\.op:\d+\.\d+\|ms|@0.939', self.logger.timing_since, 'another.op', time.time(), 0.939) self.assertStat('another.counter:3|c|@0.939', self.logger.update_stats, 'another.counter', 3, 0.939) def test_delegate_methods_with_default_sample_rate(self): self.logger = utils.get_logger({ 'log_statsd_host': 'localhost', 'log_statsd_port': str(self.port), 'log_statsd_default_sample_rate': '0.93', }, 'pfx') self.assertStat('pfx.some.counter:1|c|@0.93', self.logger.increment, 'some.counter') self.assertStat('pfx.some.counter:-1|c|@0.93', self.logger.decrement, 'some.counter') self.assertStat('pfx.some.operation:4760.0|ms|@0.93', self.logger.timing, 'some.operation', 4.76 * 1000) self.assertStatMatches('pfx\.another\.op:\d+\.\d+\|ms|@0.93', self.logger.timing_since, 'another.op', time.time()) self.assertStat('pfx.another.counter:3|c|@0.93', self.logger.update_stats, 'another.counter', 3) # Each call can override the sample_rate self.assertStat('pfx.some.counter:1|c|@0.9912', self.logger.increment, 'some.counter', sample_rate=0.9912) self.assertStat('pfx.some.counter:-1|c|@0.9912', self.logger.decrement, 'some.counter', sample_rate=0.9912) self.assertStat('pfx.some.operation:4900.0|ms|@0.9912', self.logger.timing, 'some.operation', 4.9 * 1000, sample_rate=0.9912) self.assertStatMatches('pfx\.another\.op:\d+\.\d+\|ms|@0.9912', self.logger.timing_since, 'another.op', time.time(), sample_rate=0.9912) self.assertStat('pfx.another.counter:3|c|@0.9912', self.logger.update_stats, 'another.counter', 3, sample_rate=0.9912) # Can override sample_rate with non-keyword arg self.logger.set_statsd_prefix('') self.assertStat('some.counter:1|c|@0.987654', self.logger.increment, 'some.counter', 0.987654) self.assertStat('some.counter:-1|c|@0.987654', self.logger.decrement, 'some.counter', 0.987654) self.assertStat('some.operation:4900.0|ms|@0.987654', self.logger.timing, 'some.operation', 4.9 * 1000, 0.987654) self.assertStatMatches('another\.op:\d+\.\d+\|ms|@0.987654', self.logger.timing_since, 'another.op', time.time(), 0.987654) self.assertStat('another.counter:3|c|@0.987654', self.logger.update_stats, 'another.counter', 3, 0.987654) def test_delegate_methods_with_metric_prefix(self): self.logger = utils.get_logger({ 'log_statsd_host': 'localhost', 'log_statsd_port': str(self.port), 'log_statsd_metric_prefix': 'alpha.beta', }, 'pfx') self.assertStat('alpha.beta.pfx.some.counter:1|c', self.logger.increment, 'some.counter') self.assertStat('alpha.beta.pfx.some.counter:-1|c', self.logger.decrement, 'some.counter') self.assertStat('alpha.beta.pfx.some.operation:4760.0|ms', self.logger.timing, 'some.operation', 4.76 * 1000) self.assertStatMatches( 'alpha\.beta\.pfx\.another\.op:\d+\.\d+\|ms', self.logger.timing_since, 'another.op', time.time()) self.assertStat('alpha.beta.pfx.another.counter:3|c', self.logger.update_stats, 'another.counter', 3) self.logger.set_statsd_prefix('') self.assertStat('alpha.beta.some.counter:1|c|@0.9912', self.logger.increment, 'some.counter', sample_rate=0.9912) self.assertStat('alpha.beta.some.counter:-1|c|@0.9912', self.logger.decrement, 'some.counter', 0.9912) self.assertStat('alpha.beta.some.operation:4900.0|ms|@0.9912', self.logger.timing, 'some.operation', 4.9 * 1000, sample_rate=0.9912) self.assertStatMatches('alpha\.beta\.another\.op:\d+\.\d+\|ms|@0.9912', self.logger.timing_since, 'another.op', time.time(), sample_rate=0.9912) self.assertStat('alpha.beta.another.counter:3|c|@0.9912', self.logger.update_stats, 'another.counter', 3, sample_rate=0.9912) def test_get_valid_utf8_str(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') invalid_utf8_str = unicode_sample.encode('utf-8')[::-1] self.assertEquals(valid_utf8_str, utils.get_valid_utf8_str(valid_utf8_str)) self.assertEquals(valid_utf8_str, utils.get_valid_utf8_str(unicode_sample)) self.assertEquals('\xef\xbf\xbd\xef\xbf\xbd\xec\xbc\x9d\xef\xbf\xbd', utils.get_valid_utf8_str(invalid_utf8_str)) def test_thread_locals(self): logger = utils.get_logger(None) orig_thread_locals = logger.thread_locals try: self.assertEquals(logger.thread_locals, (None, None)) logger.txn_id = '1234' logger.client_ip = '1.2.3.4' self.assertEquals(logger.thread_locals, ('1234', '1.2.3.4')) logger.txn_id = '5678' logger.client_ip = '5.6.7.8' self.assertEquals(logger.thread_locals, ('5678', '5.6.7.8')) finally: logger.thread_locals = orig_thread_locals def test_no_fdatasync(self): called = [] class NoFdatasync(object): pass def fsync(fd): called.append(fd) with patch('swift.common.utils.os', NoFdatasync()): with patch('swift.common.utils.fsync', fsync): utils.fdatasync(12345) self.assertEquals(called, [12345]) def test_yes_fdatasync(self): called = [] class YesFdatasync(object): def fdatasync(self, fd): called.append(fd) with patch('swift.common.utils.os', YesFdatasync()): utils.fdatasync(12345) self.assertEquals(called, [12345]) def test_fsync_bad_fullsync(self): class FCNTL(object): F_FULLSYNC = 123 def fcntl(self, fd, op): raise IOError(18) with patch('swift.common.utils.fcntl', FCNTL()): self.assertRaises(OSError, lambda: utils.fsync(12345)) def test_fsync_f_fullsync(self): called = [] class FCNTL(object): F_FULLSYNC = 123 def fcntl(self, fd, op): called[:] = [fd, op] return 0 with patch('swift.common.utils.fcntl', FCNTL()): utils.fsync(12345) self.assertEquals(called, [12345, 123]) def test_fsync_no_fullsync(self): called = [] class FCNTL(object): pass def fsync(fd): called.append(fd) with patch('swift.common.utils.fcntl', FCNTL()): with patch('os.fsync', fsync): utils.fsync(12345) self.assertEquals(called, [12345]) class TestThreadpool(unittest.TestCase): def _thread_id(self): return threading.current_thread().ident def _capture_args(self, *args, **kwargs): return {'args': args, 'kwargs': kwargs} def _raise_valueerror(self): return int('fishcakes') def test_run_in_thread_with_threads(self): tp = utils.ThreadPool(1) my_id = self._thread_id() other_id = tp.run_in_thread(self._thread_id) self.assertNotEquals(my_id, other_id) result = tp.run_in_thread(self._capture_args, 1, 2, bert='ernie') self.assertEquals(result, {'args': (1, 2), 'kwargs': {'bert': 'ernie'}}) caught = False try: tp.run_in_thread(self._raise_valueerror) except ValueError: caught = True self.assertTrue(caught) def test_force_run_in_thread_with_threads(self): # with nthreads > 0, force_run_in_thread looks just like run_in_thread tp = utils.ThreadPool(1) my_id = self._thread_id() other_id = tp.force_run_in_thread(self._thread_id) self.assertNotEquals(my_id, other_id) result = tp.force_run_in_thread(self._capture_args, 1, 2, bert='ernie') self.assertEquals(result, {'args': (1, 2), 'kwargs': {'bert': 'ernie'}}) self.assertRaises(ValueError, tp.force_run_in_thread, self._raise_valueerror) def test_run_in_thread_without_threads(self): # with zero threads, run_in_thread doesn't actually do so tp = utils.ThreadPool(0) my_id = self._thread_id() other_id = tp.run_in_thread(self._thread_id) self.assertEquals(my_id, other_id) result = tp.run_in_thread(self._capture_args, 1, 2, bert='ernie') self.assertEquals(result, {'args': (1, 2), 'kwargs': {'bert': 'ernie'}}) self.assertRaises(ValueError, tp.run_in_thread, self._raise_valueerror) def test_force_run_in_thread_without_threads(self): # with zero threads, force_run_in_thread uses eventlet.tpool tp = utils.ThreadPool(0) my_id = self._thread_id() other_id = tp.force_run_in_thread(self._thread_id) self.assertNotEquals(my_id, other_id) result = tp.force_run_in_thread(self._capture_args, 1, 2, bert='ernie') self.assertEquals(result, {'args': (1, 2), 'kwargs': {'bert': 'ernie'}}) self.assertRaises(ValueError, tp.force_run_in_thread, self._raise_valueerror) def test_preserving_stack_trace_from_thread(self): def gamma(): return 1 / 0 # ZeroDivisionError def beta(): return gamma() def alpha(): return beta() tp = utils.ThreadPool(1) try: tp.run_in_thread(alpha) except ZeroDivisionError: # NB: format is (filename, line number, function name, text) tb_func = [elem[2] for elem in traceback.extract_tb(sys.exc_traceback)] else: self.fail("Expected ZeroDivisionError") self.assertEqual(tb_func[-1], "gamma") self.assertEqual(tb_func[-2], "beta") self.assertEqual(tb_func[-3], "alpha") # omit the middle; what's important is that the start and end are # included, not the exact names of helper methods self.assertEqual(tb_func[1], "run_in_thread") self.assertEqual(tb_func[0], "test_preserving_stack_trace_from_thread") class TestAuditLocationGenerator(unittest.TestCase): def test_non_dir_contents(self): with temptree([]) as tmpdir: data = os.path.join(tmpdir, "drive", "data") os.makedirs(data) with open(os.path.join(data, "partition1"), "w"): pass partition = os.path.join(data, "partition2") os.makedirs(partition) with open(os.path.join(partition, "suffix1"), "w"): pass suffix = os.path.join(partition, "suffix2") os.makedirs(suffix) with open(os.path.join(suffix, "hash1"), "w"): pass locations = utils.audit_location_generator( tmpdir, "data", mount_check=False ) self.assertEqual(list(locations), []) def test_find_objects(self): with temptree([]) as tmpdir: data = os.path.join(tmpdir, "drive", "data") os.makedirs(data) partition = os.path.join(data, "partition2") os.makedirs(partition) suffix = os.path.join(partition, "suffix2") os.makedirs(suffix) hash_path = os.path.join(suffix, "hash2") os.makedirs(hash_path) obj_path = os.path.join(hash_path, "obj1.dat") with open(obj_path, "w"): pass locations = utils.audit_location_generator( tmpdir, "data", ".dat", mount_check=False ) self.assertEqual(list(locations), [(obj_path, "drive", "partition2")]) def test_ignore_metadata(self): with temptree([]) as tmpdir: data = os.path.join(tmpdir, "drive", "data") os.makedirs(data) partition = os.path.join(data, "partition2") os.makedirs(partition) suffix = os.path.join(partition, "suffix2") os.makedirs(suffix) hash_path = os.path.join(suffix, "hash2") os.makedirs(hash_path) obj_path = os.path.join(hash_path, "obj1.dat") with open(obj_path, "w"): pass meta_path = os.path.join(hash_path, "obj1.meta") with open(meta_path, "w"): pass locations = utils.audit_location_generator( tmpdir, "data", ".dat", mount_check=False ) self.assertEqual(list(locations), [(obj_path, "drive", "partition2")]) class TestGreenAsyncPile(unittest.TestCase): def test_runs_everything(self): def run_test(): tests_ran[0] += 1 return tests_ran[0] tests_ran = [0] pile = utils.GreenAsyncPile(3) for x in xrange(3): pile.spawn(run_test) self.assertEqual(sorted(x for x in pile), [1, 2, 3]) def test_is_asynchronous(self): def run_test(index): events[index].wait() return index pile = utils.GreenAsyncPile(3) for order in ((1, 2, 0), (0, 1, 2), (2, 1, 0), (0, 2, 1)): events = [eventlet.event.Event(), eventlet.event.Event(), eventlet.event.Event()] for x in xrange(3): pile.spawn(run_test, x) for x in order: events[x].send() self.assertEqual(next(pile), x) def test_next_when_empty(self): def run_test(): pass pile = utils.GreenAsyncPile(3) pile.spawn(run_test) self.assertEqual(next(pile), None) self.assertRaises(StopIteration, lambda: next(pile)) def test_waitall_timeout_timesout(self): def run_test(sleep_duration): eventlet.sleep(sleep_duration) completed[0] += 1 return sleep_duration completed = [0] pile = utils.GreenAsyncPile(3) pile.spawn(run_test, 0.1) pile.spawn(run_test, 1.0) self.assertEqual(pile.waitall(0.2), [0.1]) self.assertEqual(completed[0], 1) def test_waitall_timeout_completes(self): def run_test(sleep_duration): eventlet.sleep(sleep_duration) completed[0] += 1 return sleep_duration completed = [0] pile = utils.GreenAsyncPile(3) pile.spawn(run_test, 0.1) pile.spawn(run_test, 0.1) self.assertEqual(pile.waitall(0.5), [0.1, 0.1]) self.assertEqual(completed[0], 2) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_manager.py0000664000175400017540000021433212323703611022343 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from test.unit import temptree import os import sys import resource import signal import errno from collections import defaultdict from threading import Thread from time import sleep, time from swift.common import manager DUMMY_SIG = 1 class MockOs(object): RAISE_EPERM_SIG = 99 def __init__(self, pids): self.running_pids = pids self.pid_sigs = defaultdict(list) self.closed_fds = [] self.child_pid = 9999 # fork defaults to test parent process path self.execlp_called = False def kill(self, pid, sig): if sig == self.RAISE_EPERM_SIG: raise OSError(errno.EPERM, 'Operation not permitted') if pid not in self.running_pids: raise OSError(3, 'No such process') self.pid_sigs[pid].append(sig) def __getattr__(self, name): # I only over-ride portions of the os module try: return object.__getattr__(self, name) except AttributeError: return getattr(os, name) def pop_stream(f): """read everything out of file from the top and clear it out """ f.flush() f.seek(0) output = f.read() f.seek(0) f.truncate() #print >> sys.stderr, output return output class TestManagerModule(unittest.TestCase): def test_servers(self): main_plus_rest = set(manager.MAIN_SERVERS + manager.REST_SERVERS) self.assertEquals(set(manager.ALL_SERVERS), main_plus_rest) # make sure there's no server listed in both self.assertEquals(len(main_plus_rest), len(manager.MAIN_SERVERS) + len(manager.REST_SERVERS)) def test_setup_env(self): class MockResource(object): def __init__(self, error=None): self.error = error self.called_with_args = [] def setrlimit(self, resource, limits): if self.error: raise self.error self.called_with_args.append((resource, limits)) def __getattr__(self, name): # I only over-ride portions of the resource module try: return object.__getattr__(self, name) except AttributeError: return getattr(resource, name) _orig_resource = manager.resource _orig_environ = os.environ try: manager.resource = MockResource() manager.os.environ = {} manager.setup_env() expected = [ (resource.RLIMIT_NOFILE, (manager.MAX_DESCRIPTORS, manager.MAX_DESCRIPTORS)), (resource.RLIMIT_DATA, (manager.MAX_MEMORY, manager.MAX_MEMORY)), (resource.RLIMIT_NPROC, (manager.MAX_PROCS, manager.MAX_PROCS)), ] self.assertEquals(manager.resource.called_with_args, expected) self.assertTrue( manager.os.environ['PYTHON_EGG_CACHE'].startswith('/tmp')) # test error condition manager.resource = MockResource(error=ValueError()) manager.os.environ = {} manager.setup_env() self.assertEquals(manager.resource.called_with_args, []) self.assertTrue( manager.os.environ['PYTHON_EGG_CACHE'].startswith('/tmp')) manager.resource = MockResource(error=OSError()) manager.os.environ = {} self.assertRaises(OSError, manager.setup_env) self.assertEquals(manager.os.environ.get('PYTHON_EGG_CACHE'), None) finally: manager.resource = _orig_resource os.environ = _orig_environ def test_command_wrapper(self): @manager.command def myfunc(arg1): """test doc """ return arg1 self.assertEquals(myfunc.__doc__.strip(), 'test doc') self.assertEquals(myfunc(1), 1) self.assertEquals(myfunc(0), 0) self.assertEquals(myfunc(True), 1) self.assertEquals(myfunc(False), 0) self.assert_(hasattr(myfunc, 'publicly_accessible')) self.assert_(myfunc.publicly_accessible) def test_watch_server_pids(self): class MockOs(object): WNOHANG = os.WNOHANG def __init__(self, pid_map={}): self.pid_map = {} for pid, v in pid_map.items(): self.pid_map[pid] = (x for x in v) def waitpid(self, pid, options): try: rv = self.pid_map[pid].next() except StopIteration: raise OSError(errno.ECHILD, os.strerror(errno.ECHILD)) except KeyError: raise OSError(errno.ESRCH, os.strerror(errno.ESRCH)) if isinstance(rv, Exception): raise rv else: return rv class MockTime(object): def __init__(self, ticks=None): self.tock = time() if not ticks: ticks = [] self.ticks = (t for t in ticks) def time(self): try: self.tock += self.ticks.next() except StopIteration: self.tock += 1 return self.tock def sleep(*args): return class MockServer(object): def __init__(self, pids, run_dir=manager.RUN_DIR, zombie=0): self.heartbeat = (pids for _ in range(zombie)) def get_running_pids(self): try: rv = self.heartbeat.next() return rv except StopIteration: return {} _orig_os = manager.os _orig_time = manager.time _orig_server = manager.Server try: manager.time = MockTime() manager.os = MockOs() # this server always says it's dead when you ask for running pids server = MockServer([1]) # list of pids keyed on servers to watch server_pids = { server: [1], } # basic test, server dies gen = manager.watch_server_pids(server_pids) expected = [(server, 1)] self.assertEquals([x for x in gen], expected) # start long running server and short interval server = MockServer([1], zombie=15) server_pids = { server: [1], } gen = manager.watch_server_pids(server_pids) self.assertEquals([x for x in gen], []) # wait a little longer gen = manager.watch_server_pids(server_pids, interval=15) self.assertEquals([x for x in gen], [(server, 1)]) # zombie process server = MockServer([1], zombie=200) server_pids = { server: [1], } # test weird os error manager.os = MockOs({1: [OSError()]}) gen = manager.watch_server_pids(server_pids) self.assertRaises(OSError, lambda: [x for x in gen]) # test multi-server server1 = MockServer([1, 10], zombie=200) server2 = MockServer([2, 20], zombie=8) server_pids = { server1: [1, 10], server2: [2, 20], } pid_map = { 1: [None for _ in range(10)], 2: [None for _ in range(8)], 20: [None for _ in range(4)], } manager.os = MockOs(pid_map) gen = manager.watch_server_pids(server_pids, interval=manager.KILL_WAIT) expected = [ (server2, 2), (server2, 20), ] self.assertEquals([x for x in gen], expected) finally: manager.os = _orig_os manager.time = _orig_time manager.Server = _orig_server def test_exc(self): self.assert_(issubclass(manager.UnknownCommandError, Exception)) class TestServer(unittest.TestCase): def tearDown(self): reload(manager) def join_swift_dir(self, path): return os.path.join(manager.SWIFT_DIR, path) def join_run_dir(self, path): return os.path.join(manager.RUN_DIR, path) def test_create_server(self): server = manager.Server('proxy') self.assertEquals(server.server, 'proxy-server') self.assertEquals(server.type, 'proxy') self.assertEquals(server.cmd, 'swift-proxy-server') server = manager.Server('object-replicator') self.assertEquals(server.server, 'object-replicator') self.assertEquals(server.type, 'object') self.assertEquals(server.cmd, 'swift-object-replicator') def test_server_to_string(self): server = manager.Server('Proxy') self.assertEquals(str(server), 'proxy-server') server = manager.Server('object-replicator') self.assertEquals(str(server), 'object-replicator') def test_server_repr(self): server = manager.Server('proxy') self.assert_(server.__class__.__name__ in repr(server)) self.assert_(str(server) in repr(server)) def test_server_equality(self): server1 = manager.Server('Proxy') server2 = manager.Server('proxy-server') self.assertEquals(server1, server2) # it is NOT a string self.assertNotEquals(server1, 'proxy-server') def test_get_pid_file_name(self): server = manager.Server('proxy') conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('object-replicator') conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('container-auditor') conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) def test_get_custom_pid_file_name(self): random_run_dir = "/random/dir" get_random_run_dir = lambda x: os.path.join(random_run_dir, x) server = manager.Server('proxy', run_dir=random_run_dir) conf_file = self.join_swift_dir('proxy-server.conf') pid_file = get_random_run_dir('proxy-server.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('object-replicator', run_dir=random_run_dir) conf_file = self.join_swift_dir('object-server/1.conf') pid_file = get_random_run_dir('object-replicator/1.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) server = manager.Server('container-auditor', run_dir=random_run_dir) conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = get_random_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEquals(pid_file, server.get_pid_file_name(conf_file)) def test_get_conf_file_name(self): server = manager.Server('proxy') conf_file = self.join_swift_dir('proxy-server.conf') pid_file = self.join_run_dir('proxy-server.pid') self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('object-replicator') conf_file = self.join_swift_dir('object-server/1.conf') pid_file = self.join_run_dir('object-replicator/1.pid') self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) server = manager.Server('container-auditor') conf_file = self.join_swift_dir( 'container-server/1/container-auditor.conf') pid_file = self.join_run_dir( 'container-auditor/1/container-auditor.pid') self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) server_name = manager.STANDALONE_SERVERS[0] server = manager.Server(server_name) conf_file = self.join_swift_dir(server_name + '.conf') pid_file = self.join_run_dir(server_name + '.pid') self.assertEquals(conf_file, server.get_conf_file_name(pid_file)) def test_conf_files(self): # test get single conf file conf_files = ( 'proxy-server.conf', 'proxy-server.ini', 'auth-server.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('proxy') conf_files = server.conf_files() self.assertEquals(len(conf_files), 1) conf_file = conf_files[0] proxy_conf = self.join_swift_dir('proxy-server.conf') self.assertEquals(conf_file, proxy_conf) # test multi server conf files & grouping of server-type config conf_files = ( 'object-server1.conf', 'object-server/2.conf', 'object-server/object3.conf', 'object-server/conf/server4.conf', 'object-server.txt', 'proxy-server.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object-replicator') conf_files = server.conf_files() self.assertEquals(len(conf_files), 4) c1 = self.join_swift_dir('object-server1.conf') c2 = self.join_swift_dir('object-server/2.conf') c3 = self.join_swift_dir('object-server/object3.conf') c4 = self.join_swift_dir('object-server/conf/server4.conf') for c in [c1, c2, c3, c4]: self.assert_(c in conf_files) # test configs returned sorted sorted_confs = sorted([c1, c2, c3, c4]) self.assertEquals(conf_files, sorted_confs) # test get single numbered conf conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('account') conf_files = server.conf_files(number=2) self.assertEquals(len(conf_files), 1) conf_file = conf_files[0] self.assertEquals(conf_file, self.join_swift_dir('account-server/2.conf')) # test missing config number conf_files = server.conf_files(number=5) self.assertFalse(conf_files) # test verbose & quiet conf_files = ( 'auth-server.ini', 'container-server/1.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f server = manager.Server('auth') # check warn "unable to locate" conf_files = server.conf_files() self.assertFalse(conf_files) self.assert_('unable to locate' in pop_stream(f).lower()) # check quiet will silence warning conf_files = server.conf_files(verbose=True, quiet=True) self.assertEquals(pop_stream(f), '') # check found config no warning server = manager.Server('container-auditor') conf_files = server.conf_files() self.assertEquals(pop_stream(f), '') # check missing config number warn "unable to locate" conf_files = server.conf_files(number=2) self.assert_('unable to locate' in pop_stream(f).lower()) # check verbose lists configs conf_files = server.conf_files(number=2, verbose=True) c1 = self.join_swift_dir('container-server/1.conf') self.assert_(c1 in pop_stream(f)) finally: sys.stdout = old_stdout # test standalone conf file server_name = manager.STANDALONE_SERVERS[0] conf_files = (server_name + '.conf',) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server(server_name) conf_files = server.conf_files() self.assertEquals(len(conf_files), 1) conf_file = conf_files[0] conf = self.join_swift_dir(server_name + '.conf') self.assertEquals(conf_file, conf) def test_proxy_conf_dir(self): conf_files = ( 'proxy-server.conf.d/00.conf', 'proxy-server.conf.d/01.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('proxy') conf_dirs = server.conf_files() self.assertEquals(len(conf_dirs), 1) conf_dir = conf_dirs[0] proxy_conf_dir = self.join_swift_dir('proxy-server.conf.d') self.assertEquals(proxy_conf_dir, conf_dir) def test_conf_dir(self): conf_files = ( 'object-server/object-server.conf-base', 'object-server/1.conf.d/base.conf', 'object-server/1.conf.d/1.conf', 'object-server/2.conf.d/base.conf', 'object-server/2.conf.d/2.conf', 'object-server/3.conf.d/base.conf', 'object-server/3.conf.d/3.conf', 'object-server/4.conf.d/base.conf', 'object-server/4.conf.d/4.conf', ) with temptree(conf_files) as t: manager.SWIFT_DIR = t server = manager.Server('object-replicator') conf_dirs = server.conf_files() self.assertEquals(len(conf_dirs), 4) c1 = self.join_swift_dir('object-server/1.conf.d') c2 = self.join_swift_dir('object-server/2.conf.d') c3 = self.join_swift_dir('object-server/3.conf.d') c4 = self.join_swift_dir('object-server/4.conf.d') for c in [c1, c2, c3, c4]: self.assert_(c in conf_dirs) # test configs returned sorted sorted_confs = sorted([c1, c2, c3, c4]) self.assertEquals(conf_dirs, sorted_confs) def test_iter_pid_files(self): """ Server.iter_pid_files is kinda boring, test the Server.pid_files stuff here as well """ pid_files = ( ('proxy-server.pid', 1), ('auth-server.pid', 'blah'), ('object-replicator/1.pid', 11), ('object-replicator/2.pid', 12), ) files, contents = zip(*pid_files) with temptree(files, contents) as t: manager.RUN_DIR = t server = manager.Server('proxy', run_dir=t) # test get one file iter = server.iter_pid_files() pid_file, pid = iter.next() self.assertEquals(pid_file, self.join_run_dir('proxy-server.pid')) self.assertEquals(pid, 1) # ... and only one file self.assertRaises(StopIteration, iter.next) # test invalid value in pid file server = manager.Server('auth', run_dir=t) self.assertRaises(ValueError, server.iter_pid_files().next) # test object-server doesn't steal pids from object-replicator server = manager.Server('object', run_dir=t) self.assertRaises(StopIteration, server.iter_pid_files().next) # test multi-pid iter server = manager.Server('object-replicator', run_dir=t) real_map = { 11: self.join_run_dir('object-replicator/1.pid'), 12: self.join_run_dir('object-replicator/2.pid'), } pid_map = {} for pid_file, pid in server.iter_pid_files(): pid_map[pid] = pid_file self.assertEquals(pid_map, real_map) # test get pid_files by number conf_files = ( 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', 'object-server/4.conf', ) pid_files = ( ('object-server/1.pid', 1), ('object-server/2.pid', 2), ('object-server/5.pid', 5), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t server = manager.Server('object', run_dir=t) # test get all pid files real_map = { 1: self.join_run_dir('object-server/1.pid'), 2: self.join_run_dir('object-server/2.pid'), 5: self.join_run_dir('object-server/5.pid'), } pid_map = {} for pid_file, pid in server.iter_pid_files(): pid_map[pid] = pid_file self.assertEquals(pid_map, real_map) # test get pid with matching conf pids = list(server.iter_pid_files(number=2)) self.assertEquals(len(pids), 1) pid_file, pid = pids[0] self.assertEquals(pid, 2) pid_two = self.join_run_dir('object-server/2.pid') self.assertEquals(pid_file, pid_two) # try to iter on a pid number with a matching conf but no pid pids = list(server.iter_pid_files(number=3)) self.assertFalse(pids) # test get pids w/o matching conf pids = list(server.iter_pid_files(number=5)) self.assertFalse(pids) def test_signal_pids(self): pid_files = ( ('proxy-server.pid', 1), ('auth-server.pid', 2), ('object-server.pid', 3), ) files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # mock os with both pids running manager.os = MockOs([1, 2]) server = manager.Server('proxy', run_dir=t) pids = server.signal_pids(DUMMY_SIG) self.assertEquals(len(pids), 1) self.assert_(1 in pids) self.assertEquals(manager.os.pid_sigs[1], [DUMMY_SIG]) # make sure other process not signaled self.assertFalse(2 in pids) self.assertFalse(2 in manager.os.pid_sigs) # capture stdio old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f #test print details pids = server.signal_pids(DUMMY_SIG) output = pop_stream(f) self.assert_('pid: %s' % 1 in output) self.assert_('signal: %s' % DUMMY_SIG in output) # test no details on signal.SIG_DFL pids = server.signal_pids(signal.SIG_DFL) self.assertEquals(pop_stream(f), '') # reset mock os so only the other server is running manager.os = MockOs([2]) # test pid not running pids = server.signal_pids(signal.SIG_DFL) self.assert_(1 not in pids) self.assert_(1 not in manager.os.pid_sigs) # test remove stale pid file self.assertFalse(os.path.exists( self.join_run_dir('proxy-server.pid'))) # reset mock os with no running pids manager.os = MockOs([]) server = manager.Server('auth', run_dir=t) # test verbose warns on removing pid file pids = server.signal_pids(signal.SIG_DFL, verbose=True) output = pop_stream(f) self.assert_('stale pid' in output.lower()) auth_pid = self.join_run_dir('auth-server.pid') self.assert_(auth_pid in output) # test warning with insufficient permissions server = manager.Server('object', run_dir=t) pids = server.signal_pids(manager.os.RAISE_EPERM_SIG) output = pop_stream(f) self.assert_('no permission to signal pid 3' in output.lower(), output) finally: sys.stdout = old_stdout def test_get_running_pids(self): # test only gets running pids pid_files = ( ('test-server1.pid', 1), ('test-server2.pid', 2), ) with temptree(*zip(*pid_files)) as t: manager.RUN_DIR = t server = manager.Server('test-server', run_dir=t) # mock os, only pid '1' is running manager.os = MockOs([1]) running_pids = server.get_running_pids() self.assertEquals(len(running_pids), 1) self.assert_(1 in running_pids) self.assert_(2 not in running_pids) # test persistent running pid files self.assert_(os.path.exists(os.path.join(t, 'test-server1.pid'))) # test clean up stale pids pid_two = self.join_swift_dir('test-server2.pid') self.assertFalse(os.path.exists(pid_two)) # reset mock os, no pids running manager.os = MockOs([]) running_pids = server.get_running_pids() self.assertFalse(running_pids) # and now all pid files are cleaned out pid_one = self.join_run_dir('test-server1.pid') self.assertFalse(os.path.exists(pid_one)) all_pids = os.listdir(t) self.assertEquals(len(all_pids), 0) # test only get pids for right server pid_files = ( ('thing-doer.pid', 1), ('thing-sayer.pid', 2), ('other-doer.pid', 3), ('other-sayer.pid', 4), ) files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # all pids are running manager.os = MockOs(pids) server = manager.Server('thing-doer', run_dir=t) running_pids = server.get_running_pids() # only thing-doer.pid, 1 self.assertEquals(len(running_pids), 1) self.assert_(1 in running_pids) # no other pids returned for n in (2, 3, 4): self.assert_(n not in running_pids) # assert stale pids for other servers ignored manager.os = MockOs([1]) # only thing-doer is running running_pids = server.get_running_pids() for f in ('thing-sayer.pid', 'other-doer.pid', 'other-sayer.pid'): # other server pid files persist self.assert_(os.path.exists, os.path.join(t, f)) # verify that servers are in fact not running for server_name in ('thing-sayer', 'other-doer', 'other-sayer'): server = manager.Server(server_name, run_dir=t) running_pids = server.get_running_pids() self.assertFalse(running_pids) # and now all OTHER pid files are cleaned out all_pids = os.listdir(t) self.assertEquals(len(all_pids), 1) self.assert_(os.path.exists(os.path.join(t, 'thing-doer.pid'))) def test_kill_running_pids(self): pid_files = ( ('object-server.pid', 1), ('object-replicator1.pid', 11), ('object-replicator2.pid', 12), ) files, running_pids = zip(*pid_files) with temptree(files, running_pids) as t: manager.RUN_DIR = t server = manager.Server('object', run_dir=t) # test no servers running manager.os = MockOs([]) pids = server.kill_running_pids() self.assertFalse(pids, pids) files, running_pids = zip(*pid_files) with temptree(files, running_pids) as t: manager.RUN_DIR = t server.run_dir = t # start up pid manager.os = MockOs([1]) server = manager.Server('object', run_dir=t) # test kill one pid pids = server.kill_running_pids() self.assertEquals(len(pids), 1) self.assert_(1 in pids) self.assertEquals(manager.os.pid_sigs[1], [signal.SIGTERM]) # reset os mock manager.os = MockOs([1]) # test shutdown self.assert_('object-server' in manager.GRACEFUL_SHUTDOWN_SERVERS) pids = server.kill_running_pids(graceful=True) self.assertEquals(len(pids), 1) self.assert_(1 in pids) self.assertEquals(manager.os.pid_sigs[1], [signal.SIGHUP]) # start up other servers manager.os = MockOs([11, 12]) # test multi server kill & ignore graceful on unsupported server self.assertFalse('object-replicator' in manager.GRACEFUL_SHUTDOWN_SERVERS) server = manager.Server('object-replicator', run_dir=t) pids = server.kill_running_pids(graceful=True) self.assertEquals(len(pids), 2) for pid in (11, 12): self.assert_(pid in pids) self.assertEquals(manager.os.pid_sigs[pid], [signal.SIGTERM]) # and the other pid is of course not signaled self.assert_(1 not in manager.os.pid_sigs) def test_status(self): conf_files = ( 'test-server/1.conf', 'test-server/2.conf', 'test-server/3.conf', 'test-server/4.conf', ) pid_files = ( ('test-server/1.pid', 1), ('test-server/2.pid', 2), ('test-server/3.pid', 3), ('test-server/4.pid', 4), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # setup running servers server = manager.Server('test', run_dir=t) # capture stdio old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f # test status for all running manager.os = MockOs(pids) self.assertEquals(server.status(), 0) output = pop_stream(f).strip().splitlines() self.assertEquals(len(output), 4) for line in output: self.assert_('test-server running' in line) # test get single server by number self.assertEquals(server.status(number=4), 0) output = pop_stream(f).strip().splitlines() self.assertEquals(len(output), 1) line = output[0] self.assert_('test-server running' in line) conf_four = self.join_swift_dir(conf_files[3]) self.assert_('4 - %s' % conf_four in line) # test some servers not running manager.os = MockOs([1, 2, 3]) self.assertEquals(server.status(), 0) output = pop_stream(f).strip().splitlines() self.assertEquals(len(output), 3) for line in output: self.assert_('test-server running' in line) # test single server not running manager.os = MockOs([1, 2]) self.assertEquals(server.status(number=3), 1) output = pop_stream(f).strip().splitlines() self.assertEquals(len(output), 1) line = output[0] self.assert_('not running' in line) conf_three = self.join_swift_dir(conf_files[2]) self.assert_(conf_three in line) # test no running pids manager.os = MockOs([]) self.assertEquals(server.status(), 1) output = pop_stream(f).lower() self.assert_('no test-server running' in output) # test use provided pids pids = { 1: '1.pid', 2: '2.pid', } # shouldn't call get_running_pids called = [] def mock(*args, **kwargs): called.append(True) server.get_running_pids = mock status = server.status(pids=pids) self.assertEquals(status, 0) self.assertFalse(called) output = pop_stream(f).strip().splitlines() self.assertEquals(len(output), 2) for line in output: self.assert_('test-server running' in line) finally: sys.stdout = old_stdout def test_spawn(self): # mocks class MockProcess(object): NOTHING = 'default besides None' STDOUT = 'stdout' PIPE = 'pipe' def __init__(self, pids=None): if pids is None: pids = [] self.pids = (p for p in pids) def Popen(self, args, **kwargs): return MockProc(self.pids.next(), args, **kwargs) class MockProc(object): def __init__(self, pid, args, stdout=MockProcess.NOTHING, stderr=MockProcess.NOTHING): self.pid = pid self.args = args self.stdout = stdout if stderr == MockProcess.STDOUT: self.stderr = self.stdout else: self.stderr = stderr # setup running servers server = manager.Server('test') with temptree(['test-server.conf']) as swift_dir: manager.SWIFT_DIR = swift_dir with temptree([]) as t: manager.RUN_DIR = t server.run_dir = t old_subprocess = manager.subprocess try: # test single server process calls spawn once manager.subprocess = MockProcess([1]) conf_file = self.join_swift_dir('test-server.conf') # spawn server no kwargs server.spawn(conf_file) # test pid file pid_file = self.join_run_dir('test-server.pid') self.assert_(os.path.exists(pid_file)) pid_on_disk = int(open(pid_file).read().strip()) self.assertEquals(pid_on_disk, 1) # assert procs args self.assert_(server.procs) self.assertEquals(len(server.procs), 1) proc = server.procs[0] expected_args = [ 'swift-test-server', conf_file, ] self.assertEquals(proc.args, expected_args) # assert stdout is piped self.assertEquals(proc.stdout, MockProcess.PIPE) self.assertEquals(proc.stderr, proc.stdout) # test multi server process calls spawn multiple times manager.subprocess = MockProcess([11, 12, 13, 14]) conf1 = self.join_swift_dir('test-server/1.conf') conf2 = self.join_swift_dir('test-server/2.conf') conf3 = self.join_swift_dir('test-server/3.conf') conf4 = self.join_swift_dir('test-server/4.conf') server = manager.Server('test', run_dir=t) # test server run once server.spawn(conf1, once=True) self.assert_(server.procs) self.assertEquals(len(server.procs), 1) proc = server.procs[0] expected_args = ['swift-test-server', conf1, 'once'] # assert stdout is piped self.assertEquals(proc.stdout, MockProcess.PIPE) self.assertEquals(proc.stderr, proc.stdout) # test server not daemon server.spawn(conf2, daemon=False) self.assert_(server.procs) self.assertEquals(len(server.procs), 2) proc = server.procs[1] expected_args = ['swift-test-server', conf2, 'verbose'] self.assertEquals(proc.args, expected_args) # assert stdout is not changed self.assertEquals(proc.stdout, None) self.assertEquals(proc.stderr, None) # test server wait server.spawn(conf3, wait=False) self.assert_(server.procs) self.assertEquals(len(server.procs), 3) proc = server.procs[2] # assert stdout is /dev/null self.assert_(isinstance(proc.stdout, file)) self.assertEquals(proc.stdout.name, os.devnull) self.assertEquals(proc.stdout.mode, 'w+b') self.assertEquals(proc.stderr, proc.stdout) # test not daemon over-rides wait server.spawn(conf4, wait=False, daemon=False, once=True) self.assert_(server.procs) self.assertEquals(len(server.procs), 4) proc = server.procs[3] expected_args = ['swift-test-server', conf4, 'once', 'verbose'] self.assertEquals(proc.args, expected_args) # daemon behavior should trump wait, once shouldn't matter self.assertEquals(proc.stdout, None) self.assertEquals(proc.stderr, None) # assert pids for i, proc in enumerate(server.procs): pid_file = self.join_run_dir('test-server/%d.pid' % (i + 1)) pid_on_disk = int(open(pid_file).read().strip()) self.assertEquals(pid_on_disk, proc.pid) finally: manager.subprocess = old_subprocess def test_wait(self): server = manager.Server('test') self.assertEquals(server.wait(), 0) class MockProcess(Thread): def __init__(self, delay=0.1, fail_to_start=False): Thread.__init__(self) # setup pipe rfd, wfd = os.pipe() # subprocess connection to read stdout self.stdout = os.fdopen(rfd) # real process connection to write stdout self._stdout = os.fdopen(wfd, 'w') self.delay = delay self.finished = False self.returncode = None if fail_to_start: self._returncode = 1 self.run = self.fail else: self._returncode = 0 def __enter__(self): self.start() return self def __exit__(self, *args): if self.isAlive(): self.join() def close_stdout(self): self._stdout.flush() with open(os.devnull, 'wb') as nullfile: try: os.dup2(nullfile.fileno(), self._stdout.fileno()) except OSError: pass def fail(self): print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing print >>self._stdout, 'mock process failed to start' self.close_stdout() def poll(self): self.returncode = self._returncode return self.returncode or None def run(self): print >>self._stdout, 'mock process started' sleep(self.delay) # perform setup processing print >>self._stdout, 'setup complete!' self.close_stdout() sleep(self.delay) # do some more processing print >>self._stdout, 'mock process finished' self.finished = True class MockTime(object): def time(self): return time() def sleep(self, *args, **kwargs): pass with temptree([]) as t: old_stdout = sys.stdout old_wait = manager.WARNING_WAIT old_time = manager.time try: manager.WARNING_WAIT = 0.01 manager.time = MockTime() with open(os.path.join(t, 'output'), 'w+') as f: # acctually capture the read stdout (for prints) sys.stdout = f # test closing pipe in subprocess unblocks read with MockProcess() as proc: server.procs = [proc] status = server.wait() self.assertEquals(status, 0) # wait should return before process exits self.assert_(proc.isAlive()) self.assertFalse(proc.finished) self.assert_(proc.finished) # make sure it did finish... # test output kwarg prints subprocess output with MockProcess() as proc: server.procs = [proc] status = server.wait(output=True) output = pop_stream(f) self.assert_('mock process started' in output) self.assert_('setup complete' in output) # make sure we don't get prints after stdout was closed self.assert_('mock process finished' not in output) # test process which fails to start with MockProcess(fail_to_start=True) as proc: server.procs = [proc] status = server.wait() self.assertEquals(status, 1) self.assert_('failed' in pop_stream(f)) # test multiple procs procs = [MockProcess(delay=.5) for i in range(3)] for proc in procs: proc.start() server.procs = procs status = server.wait() self.assertEquals(status, 0) for proc in procs: self.assert_(proc.isAlive()) for proc in procs: proc.join() finally: sys.stdout = old_stdout manager.WARNING_WAIT = old_wait manager.time = old_time def test_interact(self): class MockProcess(object): def __init__(self, fail=False): self.returncode = None if fail: self._returncode = 1 else: self._returncode = 0 def communicate(self): self.returncode = self._returncode return '', '' server = manager.Server('test') server.procs = [MockProcess()] self.assertEquals(server.interact(), 0) server.procs = [MockProcess(fail=True)] self.assertEquals(server.interact(), 1) procs = [] for fail in (False, True, True): procs.append(MockProcess(fail=fail)) server.procs = procs self.assert_(server.interact() > 0) def test_launch(self): # stubs conf_files = ( 'proxy-server.conf', 'auth-server.conf', 'object-server/1.conf', 'object-server/2.conf', 'object-server/3.conf', 'object-server/4.conf', ) pid_files = ( ('proxy-server.pid', 1), ('proxy-server/2.pid', 2), ) #mocks class MockSpawn(object): def __init__(self, pids=None): self.conf_files = [] self.kwargs = [] if not pids: def one_forever(): while True: yield 1 self.pids = one_forever() else: self.pids = (x for x in pids) def __call__(self, conf_file, **kwargs): self.conf_files.append(conf_file) self.kwargs.append(kwargs) rv = self.pids.next() if isinstance(rv, Exception): raise rv else: return rv with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t old_stdout = sys.stdout try: with open(os.path.join(t, 'output'), 'w+') as f: sys.stdout = f # can't start server w/o an conf server = manager.Server('test', run_dir=t) self.assertFalse(server.launch()) # start mock os running all pids manager.os = MockOs(pids) server = manager.Server('proxy', run_dir=t) # can't start server if it's already running self.assertFalse(server.launch()) output = pop_stream(f) self.assert_('running' in output) conf_file = self.join_swift_dir('proxy-server.conf') self.assert_(conf_file in output) pid_file = self.join_run_dir('proxy-server/2.pid') self.assert_(pid_file in output) self.assert_('already started' in output) # no running pids manager.os = MockOs([]) # test ignore once for non-start-once server mock_spawn = MockSpawn([1]) server.spawn = mock_spawn conf_file = self.join_swift_dir('proxy-server.conf') expected = { 1: conf_file, } self.assertEquals(server.launch(once=True), expected) self.assertEquals(mock_spawn.conf_files, [conf_file]) expected = { 'once': False, } self.assertEquals(mock_spawn.kwargs, [expected]) output = pop_stream(f) self.assert_('Starting' in output) self.assert_('once' not in output) # test multi-server kwarg once server = manager.Server('object-replicator') mock_spawn = MockSpawn([1, 2, 3, 4]) server.spawn = mock_spawn conf1 = self.join_swift_dir('object-server/1.conf') conf2 = self.join_swift_dir('object-server/2.conf') conf3 = self.join_swift_dir('object-server/3.conf') conf4 = self.join_swift_dir('object-server/4.conf') expected = { 1: conf1, 2: conf2, 3: conf3, 4: conf4, } self.assertEquals(server.launch(once=True), expected) self.assertEquals(mock_spawn.conf_files, [ conf1, conf2, conf3, conf4]) expected = { 'once': True, } self.assertEquals(len(mock_spawn.kwargs), 4) for kwargs in mock_spawn.kwargs: self.assertEquals(kwargs, expected) # test number kwarg mock_spawn = MockSpawn([4]) server.spawn = mock_spawn expected = { 4: conf4, } self.assertEquals(server.launch(number=4), expected) self.assertEquals(mock_spawn.conf_files, [conf4]) expected = { 'number': 4 } self.assertEquals(mock_spawn.kwargs, [expected]) # test cmd does not exist server = manager.Server('auth') mock_spawn = MockSpawn([OSError(errno.ENOENT, 'blah')]) server.spawn = mock_spawn self.assertEquals(server.launch(), {}) self.assert_('swift-auth-server does not exist' in pop_stream(f)) finally: sys.stdout = old_stdout def test_stop(self): conf_files = ( 'account-server/1.conf', 'account-server/2.conf', 'account-server/3.conf', 'account-server/4.conf', ) pid_files = ( ('account-reaper/1.pid', 1), ('account-reaper/2.pid', 2), ('account-reaper/3.pid', 3), ('account-reaper/4.pid', 4), ) with temptree(conf_files) as swift_dir: manager.SWIFT_DIR = swift_dir files, pids = zip(*pid_files) with temptree(files, pids) as t: manager.RUN_DIR = t # start all pids in mock os manager.os = MockOs(pids) server = manager.Server('account-reaper', run_dir=t) # test kill all running pids pids = server.stop() self.assertEquals(len(pids), 4) for pid in (1, 2, 3, 4): self.assert_(pid in pids) self.assertEquals(manager.os.pid_sigs[pid], [signal.SIGTERM]) conf1 = self.join_swift_dir('account-reaper/1.conf') conf2 = self.join_swift_dir('account-reaper/2.conf') conf3 = self.join_swift_dir('account-reaper/3.conf') conf4 = self.join_swift_dir('account-reaper/4.conf') # reset mock os with only 2 running pids manager.os = MockOs([3, 4]) pids = server.stop() self.assertEquals(len(pids), 2) for pid in (3, 4): self.assert_(pid in pids) self.assertEquals(manager.os.pid_sigs[pid], [signal.SIGTERM]) self.assertFalse(os.path.exists(conf1)) self.assertFalse(os.path.exists(conf2)) # test number kwarg manager.os = MockOs([3, 4]) pids = server.stop(number=3) self.assertEquals(len(pids), 1) expected = { 3: conf3, } self.assert_(pids, expected) self.assertEquals(manager.os.pid_sigs[3], [signal.SIGTERM]) self.assertFalse(os.path.exists(conf4)) self.assertFalse(os.path.exists(conf3)) class TestManager(unittest.TestCase): def test_create(self): m = manager.Manager(['test']) self.assertEquals(len(m.servers), 1) server = m.servers.pop() self.assert_(isinstance(server, manager.Server)) self.assertEquals(server.server, 'test-server') # test multi-server and simple dedupe servers = ['object-replicator', 'object-auditor', 'object-replicator'] m = manager.Manager(servers) self.assertEquals(len(m.servers), 2) for server in m.servers: self.assert_(server.server in servers) # test all m = manager.Manager(['all']) self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) # test main m = manager.Manager(['main']) self.assertEquals(len(m.servers), len(manager.MAIN_SERVERS)) for server in m.servers: self.assert_(server.server in manager.MAIN_SERVERS) # test rest m = manager.Manager(['rest']) self.assertEquals(len(m.servers), len(manager.REST_SERVERS)) for server in m.servers: self.assert_(server.server in manager.REST_SERVERS) # test main + rest == all m = manager.Manager(['main', 'rest']) self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) # test dedupe m = manager.Manager(['main', 'rest', 'proxy', 'object', 'container', 'account']) self.assertEquals(len(m.servers), len(manager.ALL_SERVERS)) for server in m.servers: self.assert_(server.server in manager.ALL_SERVERS) # test glob m = manager.Manager(['object-*']) object_servers = [s for s in manager.ALL_SERVERS if s.startswith('object')] self.assertEquals(len(m.servers), len(object_servers)) for s in m.servers: self.assert_(str(s) in object_servers) m = manager.Manager(['*-replicator']) replicators = [s for s in manager.ALL_SERVERS if s.endswith('replicator')] for s in m.servers: self.assert_(str(s) in replicators) def test_iter(self): m = manager.Manager(['all']) self.assertEquals(len(list(m)), len(manager.ALL_SERVERS)) for server in m: self.assert_(server.server in manager.ALL_SERVERS) def test_status(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called_kwargs = [] def status(self, **kwargs): self.called_kwargs.append(kwargs) if 'error' in self.server: return 1 else: return 0 old_server_class = manager.Server try: manager.Server = MockServer m = manager.Manager(['test']) status = m.status() self.assertEquals(status, 0) m = manager.Manager(['error']) status = m.status() self.assertEquals(status, 1) # test multi-server m = manager.Manager(['test', 'error']) kwargs = {'key': 'value'} status = m.status(**kwargs) self.assertEquals(status, 1) for server in m.servers: self.assertEquals(server.called_kwargs, [kwargs]) finally: manager.Server = old_server_class def test_start(self): def mock_setup_env(): getattr(mock_setup_env, 'called', []).append(True) class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) def wait(self, **kwargs): self.called['wait'].append(kwargs) return int('error' in self.server) def stop(self, **kwargs): self.called['stop'].append(kwargs) def interact(self, **kwargs): self.called['interact'].append(kwargs) if 'raise' in self.server: raise KeyboardInterrupt elif 'error' in self.server: return 1 else: return 0 old_setup_env = manager.setup_env old_swift_server = manager.Server try: manager.setup_env = mock_setup_env manager.Server = MockServer # test no errors on launch m = manager.Manager(['proxy']) status = m.start() self.assertEquals(status, 0) for server in m.servers: self.assertEquals(server.called['launch'], [{}]) # test error on launch m = manager.Manager(['proxy', 'error']) status = m.start() self.assertEquals(status, 1) for server in m.servers: self.assertEquals(server.called['launch'], [{}]) self.assertEquals(server.called['wait'], [{}]) # test interact m = manager.Manager(['proxy', 'error']) kwargs = {'daemon': False} status = m.start(**kwargs) self.assertEquals(status, 1) for server in m.servers: self.assertEquals(server.called['launch'], [kwargs]) self.assertEquals(server.called['interact'], [kwargs]) m = manager.Manager(['raise']) kwargs = {'daemon': False} status = m.start(**kwargs) finally: manager.setup_env = old_setup_env manager.Server = old_swift_server def test_no_wait(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) def wait(self, **kwargs): self.called['wait'].append(kwargs) return int('error' in self.server) orig_swift_server = manager.Server try: manager.Server = MockServer # test success init = manager.Manager(['proxy']) status = init.no_wait() self.assertEquals(status, 0) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertFalse(called_kwargs['wait']) self.assertFalse(server.called['wait']) # test no errocode status even on error init = manager.Manager(['error']) status = init.no_wait() self.assertEquals(status, 0) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assert_('wait' in called_kwargs) self.assertFalse(called_kwargs['wait']) self.assertFalse(server.called['wait']) # test wait with once option init = manager.Manager(['updater', 'replicator-error']) status = init.no_wait(once=True) self.assertEquals(status, 0) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assert_('wait' in called_kwargs) self.assertFalse(called_kwargs['wait']) self.assert_('once' in called_kwargs) self.assert_(called_kwargs['once']) self.assertFalse(server.called['wait']) finally: manager.Server = orig_swift_server def test_no_daemon(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def launch(self, **kwargs): self.called['launch'].append(kwargs) def interact(self, **kwargs): self.called['interact'].append(kwargs) return int('error' in self.server) orig_swift_server = manager.Server try: manager.Server = MockServer # test success init = manager.Manager(['proxy']) stats = init.no_daemon() self.assertEquals(stats, 0) # test error init = manager.Manager(['proxy', 'object-error']) stats = init.no_daemon() self.assertEquals(stats, 1) # test once init = manager.Manager(['proxy', 'object-error']) stats = init.no_daemon() for server in init.servers: self.assertEquals(len(server.called['launch']), 1) self.assertEquals(len(server.called['wait']), 0) self.assertEquals(len(server.called['interact']), 1) finally: manager.Server = orig_swift_server def test_once(self): class MockServer(object): def __init__(self, server, run_dir=manager.RUN_DIR): self.server = server self.called = defaultdict(list) def wait(self, **kwargs): self.called['wait'].append(kwargs) if 'error' in self.server: return 1 else: return 0 def launch(self, **kwargs): return self.called['launch'].append(kwargs) orig_swift_server = manager.Server try: manager.Server = MockServer # test no errors init = manager.Manager(['account-reaper']) status = init.once() self.assertEquals(status, 0) # test error code on error init = manager.Manager(['error-reaper']) status = init.once() self.assertEquals(status, 1) for server in init.servers: self.assertEquals(len(server.called['launch']), 1) called_kwargs = server.called['launch'][0] self.assertEquals(called_kwargs, {'once': True}) self.assertEquals(len(server.called['wait']), 1) self.assertEquals(len(server.called['interact']), 0) finally: manager.Server = orig_swift_server def test_stop(self): class MockServerFactory(object): class MockServer(object): def __init__(self, pids, run_dir=manager.RUN_DIR): self.pids = pids def stop(self, **kwargs): return self.pids def status(self, **kwargs): return not self.pids def __init__(self, server_pids, run_dir=manager.RUN_DIR): self.server_pids = server_pids def __call__(self, server, run_dir=manager.RUN_DIR): return MockServerFactory.MockServer(self.server_pids[server]) def mock_watch_server_pids(server_pids, **kwargs): for server, pids in server_pids.items(): for pid in pids: if pid is None: continue yield server, pid _orig_server = manager.Server _orig_watch_server_pids = manager.watch_server_pids try: manager.watch_server_pids = mock_watch_server_pids # test stop one server server_pids = { 'test': [1] } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEquals(status, 0) # test not running server_pids = { 'test': [] } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEquals(status, 1) # test kill not running server_pids = { 'test': [] } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.kill() self.assertEquals(status, 0) # test won't die server_pids = { 'test': [None] } manager.Server = MockServerFactory(server_pids) m = manager.Manager(['test']) status = m.stop() self.assertEquals(status, 1) finally: manager.Server = _orig_server manager.watch_server_pids = _orig_watch_server_pids # TODO(clayg): more tests def test_shutdown(self): m = manager.Manager(['test']) m.stop_was_called = False def mock_stop(*args, **kwargs): m.stop_was_called = True expected = {'graceful': True} self.assertEquals(kwargs, expected) return 0 m.stop = mock_stop status = m.shutdown() self.assertEquals(status, 0) self.assertEquals(m.stop_was_called, True) def test_restart(self): m = manager.Manager(['test']) m.stop_was_called = False def mock_stop(*args, **kwargs): m.stop_was_called = True return 0 m.start_was_called = False def mock_start(*args, **kwargs): m.start_was_called = True return 0 m.stop = mock_stop m.start = mock_start status = m.restart() self.assertEquals(status, 0) self.assertEquals(m.stop_was_called, True) self.assertEquals(m.start_was_called, True) def test_reload(self): class MockManager(object): called = defaultdict(list) def __init__(self, servers): pass @classmethod def reset_called(cls): cls.called = defaultdict(list) def stop(self, **kwargs): MockManager.called['stop'].append(kwargs) return 0 def start(self, **kwargs): MockManager.called['start'].append(kwargs) return 0 _orig_manager = manager.Manager try: m = _orig_manager(['auth']) for server in m.servers: self.assert_(server.server in manager.GRACEFUL_SHUTDOWN_SERVERS) manager.Manager = MockManager status = m.reload() self.assertEquals(status, 0) expected = { 'start': [{'graceful': True}], 'stop': [{'graceful': True}], } self.assertEquals(MockManager.called, expected) # test force graceful MockManager.reset_called() m = _orig_manager(['*-server']) self.assertEquals(len(m.servers), 4) for server in m.servers: self.assert_(server.server in manager.GRACEFUL_SHUTDOWN_SERVERS) manager.Manager = MockManager status = m.reload(graceful=False) self.assertEquals(status, 0) expected = { 'start': [{'graceful': True}] * 4, 'stop': [{'graceful': True}] * 4, } self.assertEquals(MockManager.called, expected) finally: manager.Manager = _orig_manager def test_force_reload(self): m = manager.Manager(['test']) m.reload_was_called = False def mock_reload(*args, **kwargs): m.reload_was_called = True return 0 m.reload = mock_reload status = m.force_reload() self.assertEquals(status, 0) self.assertEquals(m.reload_was_called, True) def test_get_command(self): m = manager.Manager(['test']) self.assertEquals(m.start, m.get_command('start')) self.assertEquals(m.force_reload, m.get_command('force-reload')) self.assertEquals(m.get_command('force-reload'), m.get_command('force_reload')) self.assertRaises(manager.UnknownCommandError, m.get_command, 'no_command') self.assertRaises(manager.UnknownCommandError, m.get_command, '__init__') def test_list_commands(self): for cmd, help in manager.Manager.list_commands(): method = getattr(manager.Manager, cmd.replace('-', '_'), None) self.assert_(method, '%s is not a command' % cmd) self.assert_(getattr(method, 'publicly_accessible', False)) self.assertEquals(method.__doc__.strip(), help) def test_run_command(self): m = manager.Manager(['test']) m.cmd_was_called = False def mock_cmd(*args, **kwargs): m.cmd_was_called = True expected = {'kw1': True, 'kw2': False} self.assertEquals(kwargs, expected) return 0 mock_cmd.publicly_accessible = True m.mock_cmd = mock_cmd kwargs = {'kw1': True, 'kw2': False} status = m.run_command('mock_cmd', **kwargs) self.assertEquals(status, 0) self.assertEquals(m.cmd_was_called, True) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_bufferedhttp.py0000664000175400017540000000763212323703611023416 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from eventlet import spawn, Timeout, listen from swift.common import bufferedhttp class TestBufferedHTTP(unittest.TestCase): def test_http_connect(self): bindsock = listen(('127.0.0.1', 0)) def accept(expected_par): try: with Timeout(3): sock, addr = bindsock.accept() fp = sock.makefile() fp.write('HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\n' 'RESPONSE') fp.flush() self.assertEquals( fp.readline(), 'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' % expected_par) headers = {} line = fp.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = fp.readline() self.assertEquals(headers['content-length'], '7') self.assertEquals(headers['x-header'], 'value') self.assertEquals(fp.readline(), 'REQUEST\r\n') except BaseException as err: return err return None for par in ('par', 1357): event = spawn(accept, par) try: with Timeout(3): conn = bufferedhttp.http_connect( '127.0.0.1', bindsock.getsockname()[1], 'dev', par, 'PUT', '/path/..%/', { 'content-length': 7, 'x-header': 'value'}, query_string='omg&no=%7f') conn.send('REQUEST\r\n') resp = conn.getresponse() body = resp.read() conn.close() self.assertEquals(resp.status, 200) self.assertEquals(resp.reason, 'OK') self.assertEquals(body, 'RESPONSE') finally: err = event.wait() if err: raise Exception(err) def test_nonstr_header_values(self): class MockHTTPSConnection(object): def __init__(self, hostport): pass def putrequest(self, method, path, skip_host=0): pass def putheader(self, header, *values): # Essentially what Python 2.7 does that caused us problems. '\r\n\t'.join(values) def endheaders(self): pass origHTTPSConnection = bufferedhttp.HTTPSConnection bufferedhttp.HTTPSConnection = MockHTTPSConnection try: bufferedhttp.http_connect( '127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) bufferedhttp.http_connect_raw( '127.0.0.1', 8080, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) finally: bufferedhttp.HTTPSConnection = origHTTPSConnection if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/test_exceptions.py0000664000175400017540000000364712323703611023117 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # TODO(creiht): Tests import unittest from swift.common import exceptions class TestExceptions(unittest.TestCase): def test_replication_exception(self): self.assertEqual(str(exceptions.ReplicationException()), '') self.assertEqual(str(exceptions.ReplicationException('test')), 'test') def test_replication_lock_timeout(self): exc = exceptions.ReplicationLockTimeout(15, 'test') try: self.assertTrue(isinstance(exc, exceptions.MessageTimeout)) finally: exc.cancel() def test_client_exception(self): strerror = 'test: HTTP://random:888/randompath?foo=1 666 reason: ' \ 'device /sdb1 content' exc = exceptions.ClientException('test', http_scheme='HTTP', http_host='random', http_port=888, http_path='/randompath', http_query='foo=1', http_status=666, http_reason='reason', http_device='/sdb1', http_response_content='content') self.assertEqual(str(exc), strerror) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/malformed_example.db0000664000175400017540000002000012323703611023273 0ustar jenkinsjenkins00000000000000SQLite format 3j@      F''Ktableoutgoing_syncoutgoing_syncCREATE TABLE outgoing_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 )9M'indexsqlite_autoindex_outgoing_sync_1outgoing_syncF''Ktableincoming_syncincoming_syncCREATE TABLE incoming_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 )9M'indexsqlite_autoindex_incoming_sync_1incoming_sync5']triggeroutgoing_sync_insertoutgoing_syncCREATE TRIGGER outgoing_sync_insert AFTER INSERT ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END }}0 EtabletesttestCREATE TABLE test (one TEXT)5']triggerincoming_sync_updateincoming_syncCREATE TRIGGER incoming_sync_update AFTER UPDATE ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END5']triggerincoming_sync_insertincoming_syncCREATE TRIGGER incoming_sync_insert AFTER INSERT ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END5']triggeroutgoing_sync_updateoutgoing_syncCREATE TRIGGER outgoing_sync_update AFTER UPDATE ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END 1swift-1.13.1/test/unit/common/test_db_replicator.py0000664000175400017540000014401312323703611023540 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from contextlib import contextmanager import os import logging import errno import math import time from mock import patch, call from shutil import rmtree from tempfile import mkdtemp, NamedTemporaryFile import mock import simplejson from swift.container.backend import DATADIR from swift.common import db_replicator from swift.common.utils import normalize_timestamp from swift.common.exceptions import DriveNotMounted from swift.common.swob import HTTPException from test.unit import FakeLogger TEST_ACCOUNT_NAME = 'a c t' TEST_CONTAINER_NAME = 'c o n' def teardown_module(): "clean up my monkey patching" reload(db_replicator) @contextmanager def lock_parent_directory(filename): yield True class FakeRing(object): class Ring(object): devs = [] def __init__(self, path, reload_time=15, ring_name=None): pass def get_part(self, account, container=None, obj=None): return 0 def get_part_nodes(self, part): return [] def get_more_nodes(self, *args): return [] class FakeRingWithSingleNode(object): class Ring(object): devs = [dict( id=1, weight=10.0, zone=1, ip='1.1.1.1', port=6000, device='sdb', meta='', replication_ip='1.1.1.1', replication_port=6000 )] def __init__(self, path, reload_time=15, ring_name=None): pass def get_part(self, account, container=None, obj=None): return 0 def get_part_nodes(self, part): return self.devs def get_more_nodes(self, *args): return (d for d in self.devs) class FakeRingWithNodes(object): class Ring(object): devs = [dict( id=1, weight=10.0, zone=1, ip='1.1.1.1', port=6000, device='sdb', meta='' ), dict( id=2, weight=10.0, zone=2, ip='1.1.1.2', port=6000, device='sdb', meta='' ), dict( id=3, weight=10.0, zone=3, ip='1.1.1.3', port=6000, device='sdb', meta='' ), dict( id=4, weight=10.0, zone=4, ip='1.1.1.4', port=6000, device='sdb', meta='' ), dict( id=5, weight=10.0, zone=5, ip='1.1.1.5', port=6000, device='sdb', meta='' ), dict( id=6, weight=10.0, zone=6, ip='1.1.1.6', port=6000, device='sdb', meta='')] def __init__(self, path, reload_time=15, ring_name=None): pass def get_part(self, account, container=None, obj=None): return 0 def get_part_nodes(self, part): return self.devs[:3] def get_more_nodes(self, *args): return (d for d in self.devs[3:]) class FakeProcess(object): def __init__(self, *codes): self.codes = iter(codes) self.args = None self.kwargs = None def __call__(self, *args, **kwargs): self.args = args self.kwargs = kwargs class Failure(object): def communicate(innerself): next = self.codes.next() if isinstance(next, int): innerself.returncode = next return next raise next return Failure() @contextmanager def _mock_process(*args): orig_process = db_replicator.subprocess.Popen db_replicator.subprocess.Popen = FakeProcess(*args) yield db_replicator.subprocess.Popen db_replicator.subprocess.Popen = orig_process class ReplHttp(object): def __init__(self, response=None, set_status=200): self.response = response self.set_status = set_status replicated = False host = 'localhost' def replicate(self, *args): self.replicated = True class Response(object): status = self.set_status data = self.response def read(innerself): return self.response return Response() class ChangingMtimesOs(object): def __init__(self): self.mtime = 0 def __call__(self, *args, **kwargs): self.mtime += 1 return self.mtime class FakeBroker(object): db_file = __file__ get_repl_missing_table = False stub_replication_info = None db_type = 'container' info = {'account': TEST_ACCOUNT_NAME, 'container': TEST_CONTAINER_NAME} def __init__(self, *args, **kwargs): self.locked = False return None @contextmanager def lock(self): self.locked = True yield True self.locked = False def get_sync(self, *args, **kwargs): return 5 def get_syncs(self): return [] def get_items_since(self, point, *args): if point == 0: return [{'ROWID': 1}] if point == -1: return [{'ROWID': 1}, {'ROWID': 2}] return [] def merge_syncs(self, *args, **kwargs): self.args = args def merge_items(self, *args): self.args = args def get_replication_info(self): if self.get_repl_missing_table: raise Exception('no such table') if self.stub_replication_info: return self.stub_replication_info return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0, 'hash': 12345} def reclaim(self, item_timestamp, sync_timestamp): pass def get_info(self): return self.info def newid(self, remote_d): pass def update_metadata(self, metadata): self.metadata = metadata def merge_timestamps(self, created_at, put_timestamp, delete_timestamp): self.created_at = created_at self.put_timestamp = put_timestamp self.delete_timestamp = delete_timestamp class FakeAccountBroker(FakeBroker): db_type = 'account' info = {'account': TEST_ACCOUNT_NAME} class TestReplicator(db_replicator.Replicator): server_type = 'container' ring_file = 'container.ring.gz' brokerclass = FakeBroker datadir = DATADIR default_port = 1000 class TestDBReplicator(unittest.TestCase): def setUp(self): db_replicator.ring = FakeRing() self.delete_db_calls = [] self._patchers = [] def tearDown(self): for patcher in self._patchers: patcher.stop() def _patch(self, patching_fn, *args, **kwargs): patcher = patching_fn(*args, **kwargs) patched_thing = patcher.start() self._patchers.append(patcher) return patched_thing def stub_delete_db(self, object_file): self.delete_db_calls.append(object_file) def test_repl_connection(self): node = {'replication_ip': '127.0.0.1', 'replication_port': 80, 'device': 'sdb1'} conn = db_replicator.ReplConnection(node, '1234567890', 'abcdefg', logging.getLogger()) def req(method, path, body, headers): self.assertEquals(method, 'REPLICATE') self.assertEquals(headers['Content-Type'], 'application/json') class Resp(object): def read(self): return 'data' resp = Resp() conn.request = req conn.getresponse = lambda *args: resp self.assertEquals(conn.replicate(1, 2, 3), resp) def other_req(method, path, body, headers): raise Exception('blah') conn.request = other_req self.assertEquals(conn.replicate(1, 2, 3), None) def test_rsync_file(self): replicator = TestReplicator({}) with _mock_process(-1): self.assertEquals( False, replicator._rsync_file('/some/file', 'remote:/some/file')) with _mock_process(0): self.assertEquals( True, replicator._rsync_file('/some/file', 'remote:/some/file')) def test_rsync_file_popen_args(self): replicator = TestReplicator({}) with _mock_process(0) as process: replicator._rsync_file('/some/file', 'remote:/some_file') exp_args = ([ 'rsync', '--quiet', '--no-motd', '--timeout=%s' % int(math.ceil(replicator.node_timeout)), '--contimeout=%s' % int(math.ceil(replicator.conn_timeout)), '--whole-file', '/some/file', 'remote:/some_file'],) self.assertEqual(exp_args, process.args) def test_rsync_file_popen_args_whole_file_false(self): replicator = TestReplicator({}) with _mock_process(0) as process: replicator._rsync_file('/some/file', 'remote:/some_file', False) exp_args = ([ 'rsync', '--quiet', '--no-motd', '--timeout=%s' % int(math.ceil(replicator.node_timeout)), '--contimeout=%s' % int(math.ceil(replicator.conn_timeout)), '/some/file', 'remote:/some_file'],) self.assertEqual(exp_args, process.args) def test_rsync_db(self): replicator = TestReplicator({}) replicator._rsync_file = lambda *args: True fake_device = {'replication_ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(FakeBroker(), fake_device, ReplHttp(), 'abcd') def test_rsync_db_rsync_file_call(self): fake_device = {'ip': '127.0.0.1', 'port': '0', 'replication_ip': '127.0.0.1', 'replication_port': '0', 'device': 'sda1'} def mock_rsync_ip(ip): self.assertEquals(fake_device['ip'], ip) return 'rsync_ip(%s)' % ip class MyTestReplicator(TestReplicator): def __init__(self, db_file, remote_file): super(MyTestReplicator, self).__init__({}) self.db_file = db_file self.remote_file = remote_file def _rsync_file(self_, db_file, remote_file, whole_file=True): self.assertEqual(self_.db_file, db_file) self.assertEqual(self_.remote_file, remote_file) self_._rsync_file_called = True return False with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): broker = FakeBroker() remote_file = 'rsync_ip(127.0.0.1)::container/sda1/tmp/abcd' replicator = MyTestReplicator(broker.db_file, remote_file) replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') self.assert_(replicator._rsync_file_called) with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): broker = FakeBroker() remote_file = 'rsync_ip(127.0.0.1)::container0/sda1/tmp/abcd' replicator = MyTestReplicator(broker.db_file, remote_file) replicator.vm_test_mode = True replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') self.assert_(replicator._rsync_file_called) def test_rsync_db_rsync_file_failure(self): class MyTestReplicator(TestReplicator): def __init__(self): super(MyTestReplicator, self).__init__({}) self._rsync_file_called = False def _rsync_file(self_, *args, **kwargs): self.assertEqual( False, self_._rsync_file_called, '_sync_file() should only be called once') self_._rsync_file_called = True return False with patch('os.path.exists', lambda *args: True): replicator = MyTestReplicator() fake_device = {'ip': '127.0.0.1', 'replication_ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(FakeBroker(), fake_device, ReplHttp(), 'abcd') self.assertEqual(True, replicator._rsync_file_called) def test_rsync_db_change_after_sync(self): class MyTestReplicator(TestReplicator): def __init__(self, broker): super(MyTestReplicator, self).__init__({}) self.broker = broker self._rsync_file_call_count = 0 def _rsync_file(self_, db_file, remote_file, whole_file=True): self_._rsync_file_call_count += 1 if self_._rsync_file_call_count == 1: self.assertEquals(True, whole_file) self.assertEquals(False, self_.broker.locked) elif self_._rsync_file_call_count == 2: self.assertEquals(False, whole_file) self.assertEquals(True, self_.broker.locked) else: raise RuntimeError('_rsync_file() called too many times') return True # with journal file with patch('os.path.exists', lambda *args: True): broker = FakeBroker() replicator = MyTestReplicator(broker) fake_device = {'ip': '127.0.0.1', 'replication_ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') self.assertEquals(2, replicator._rsync_file_call_count) # with new mtime with patch('os.path.exists', lambda *args: False): with patch('os.path.getmtime', ChangingMtimesOs()): broker = FakeBroker() replicator = MyTestReplicator(broker) fake_device = {'ip': '127.0.0.1', 'replication_ip': '127.0.0.1', 'device': 'sda1'} replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') self.assertEquals(2, replicator._rsync_file_call_count) def test_in_sync(self): replicator = TestReplicator({}) self.assertEquals(replicator._in_sync( {'id': 'a', 'point': 0, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, FakeBroker(), -1), True) self.assertEquals(replicator._in_sync( {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'}, {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'b'}, FakeBroker(), -1), True) self.assertEquals(bool(replicator._in_sync( {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'c'}, {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'd'}, FakeBroker(), -1)), False) def test_run_once(self): replicator = TestReplicator({}) replicator.run_once() def test_run_once_no_ips(self): replicator = TestReplicator({}) replicator.logger = FakeLogger() self._patch(patch.object, db_replicator, 'whataremyips', lambda *args: []) replicator.run_once() self.assertEqual( replicator.logger.log_dict['error'], [(('ERROR Failed to get my own IPs?',), {})]) def test_run_once_node_is_not_mounted(self): db_replicator.ring = FakeRingWithSingleNode() replicator = TestReplicator({}) replicator.logger = FakeLogger() replicator.mount_check = True replicator.port = 6000 def mock_ismount(path): self.assertEquals(path, os.path.join(replicator.root, replicator.ring.devs[0]['device'])) return False self._patch(patch.object, db_replicator, 'whataremyips', lambda *args: ['1.1.1.1']) self._patch(patch.object, db_replicator, 'ismount', mock_ismount) replicator.run_once() self.assertEqual( replicator.logger.log_dict['warning'], [(('Skipping %(device)s as it is not mounted' % replicator.ring.devs[0],), {})]) def test_run_once_node_is_mounted(self): db_replicator.ring = FakeRingWithSingleNode() replicator = TestReplicator({}) replicator.logger = FakeLogger() replicator.mount_check = True replicator.port = 6000 def mock_unlink_older_than(path, mtime): self.assertEquals(path, os.path.join(replicator.root, replicator.ring.devs[0]['device'], 'tmp')) self.assertTrue(time.time() - replicator.reclaim_age >= mtime) def mock_spawn_n(fn, part, object_file, node_id): self.assertEquals('123', part) self.assertEquals('/srv/node/sda/c.db', object_file) self.assertEquals(1, node_id) self._patch(patch.object, db_replicator, 'whataremyips', lambda *args: ['1.1.1.1']) self._patch(patch.object, db_replicator, 'ismount', lambda *args: True) self._patch(patch.object, db_replicator, 'unlink_older_than', mock_unlink_older_than) self._patch(patch.object, db_replicator, 'roundrobin_datadirs', lambda *args: [('123', '/srv/node/sda/c.db', 1)]) self._patch(patch.object, replicator.cpool, 'spawn_n', mock_spawn_n) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.isdir.return_value = True replicator.run_once() mock_os.path.isdir.assert_called_with( os.path.join(replicator.root, replicator.ring.devs[0]['device'], replicator.datadir)) def test_usync(self): fake_http = ReplHttp() replicator = TestReplicator({}) replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890') def test_usync_http_error_above_300(self): fake_http = ReplHttp(set_status=301) replicator = TestReplicator({}) self.assertFalse( replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')) def test_usync_http_error_below_200(self): fake_http = ReplHttp(set_status=101) replicator = TestReplicator({}) self.assertFalse( replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')) def test_stats(self): # I'm not sure how to test that this logs the right thing, # but we can at least make sure it gets covered. replicator = TestReplicator({}) replicator._zero_stats() replicator._report_stats() def test_replicate_object(self): db_replicator.ring = FakeRingWithNodes() replicator = TestReplicator({}) replicator.delete_db = self.stub_delete_db replicator._replicate_object('0', '/path/to/file', 'node_id') self.assertEquals([], self.delete_db_calls) def test_replicate_object_quarantine(self): replicator = TestReplicator({}) self._patch(patch.object, replicator.brokerclass, 'db_file', '/a/b/c/d/e/hey') self._patch(patch.object, replicator.brokerclass, 'get_repl_missing_table', True) def mock_renamer(was, new, cause_colision=False): if cause_colision and '-' not in new: raise OSError(errno.EEXIST, "File already exists") self.assertEquals('/a/b/c/d/e', was) if '-' in new: self.assert_( new.startswith('/a/quarantined/containers/e-')) else: self.assertEquals('/a/quarantined/containers/e', new) def mock_renamer_error(was, new): return mock_renamer(was, new, cause_colision=True) with patch.object(db_replicator, 'renamer', mock_renamer): replicator._replicate_object('0', 'file', 'node_id') # try the double quarantine with patch.object(db_replicator, 'renamer', mock_renamer_error): replicator._replicate_object('0', 'file', 'node_id') def test_replicate_object_delete_because_deleted(self): replicator = TestReplicator({}) try: replicator.delete_db = self.stub_delete_db replicator.brokerclass.stub_replication_info = { 'delete_timestamp': 2, 'put_timestamp': 1, 'count': 0} replicator._replicate_object('0', '/path/to/file', 'node_id') finally: replicator.brokerclass.stub_replication_info = None self.assertEquals(['/path/to/file'], self.delete_db_calls) def test_replicate_object_delete_because_not_shouldbehere(self): replicator = TestReplicator({}) replicator.delete_db = self.stub_delete_db replicator._replicate_object('0', '/path/to/file', 'node_id') self.assertEquals(['/path/to/file'], self.delete_db_calls) def test_replicate_account_out_of_place(self): replicator = TestReplicator({}) replicator.ring = FakeRingWithNodes().Ring('path') replicator.brokerclass = FakeAccountBroker replicator._repl_to_node = lambda *args: True replicator.delete_db = self.stub_delete_db replicator.logger = FakeLogger() # Correct node_id, wrong part part = replicator.ring.get_part(TEST_ACCOUNT_NAME) + 1 node_id = replicator.ring.get_part_nodes(part)[0]['id'] replicator._replicate_object(str(part), '/path/to/file', node_id) self.assertEqual(['/path/to/file'], self.delete_db_calls) self.assertEqual( replicator.logger.log_dict['error'], [(('Found /path/to/file for /a%20c%20t when it should be on ' 'partition 0; will replicate out and remove.',), {})]) def test_replicate_container_out_of_place(self): replicator = TestReplicator({}) replicator.ring = FakeRingWithNodes().Ring('path') replicator._repl_to_node = lambda *args: True replicator.delete_db = self.stub_delete_db replicator.logger = FakeLogger() # Correct node_id, wrong part part = replicator.ring.get_part( TEST_ACCOUNT_NAME, TEST_CONTAINER_NAME) + 1 node_id = replicator.ring.get_part_nodes(part)[0]['id'] replicator._replicate_object(str(part), '/path/to/file', node_id) self.assertEqual(['/path/to/file'], self.delete_db_calls) self.assertEqual( replicator.logger.log_dict['error'], [(('Found /path/to/file for /a%20c%20t/c%20o%20n when it should ' 'be on partition 0; will replicate out and remove.',), {})]) def test_delete_db(self): db_replicator.lock_parent_directory = lock_parent_directory replicator = TestReplicator({}) replicator._zero_stats() replicator.extract_device = lambda _: 'some_device' replicator.logger = FakeLogger() temp_dir = mkdtemp() try: temp_suf_dir = os.path.join(temp_dir, '16e') os.mkdir(temp_suf_dir) temp_hash_dir = os.path.join(temp_suf_dir, '166e33924a08ede4204871468c11e16e') os.mkdir(temp_hash_dir) temp_file = NamedTemporaryFile(dir=temp_hash_dir, delete=False) temp_hash_dir2 = os.path.join(temp_suf_dir, '266e33924a08ede4204871468c11e16e') os.mkdir(temp_hash_dir2) temp_file2 = NamedTemporaryFile(dir=temp_hash_dir2, delete=False) # sanity-checks self.assertTrue(os.path.exists(temp_dir)) self.assertTrue(os.path.exists(temp_suf_dir)) self.assertTrue(os.path.exists(temp_hash_dir)) self.assertTrue(os.path.exists(temp_file.name)) self.assertTrue(os.path.exists(temp_hash_dir2)) self.assertTrue(os.path.exists(temp_file2.name)) self.assertEqual(0, replicator.stats['remove']) replicator.delete_db(temp_file.name) self.assertTrue(os.path.exists(temp_dir)) self.assertTrue(os.path.exists(temp_suf_dir)) self.assertFalse(os.path.exists(temp_hash_dir)) self.assertFalse(os.path.exists(temp_file.name)) self.assertTrue(os.path.exists(temp_hash_dir2)) self.assertTrue(os.path.exists(temp_file2.name)) self.assertEqual([(('removes.some_device',), {})], replicator.logger.log_dict['increment']) self.assertEqual(1, replicator.stats['remove']) replicator.delete_db(temp_file2.name) self.assertTrue(os.path.exists(temp_dir)) self.assertFalse(os.path.exists(temp_suf_dir)) self.assertFalse(os.path.exists(temp_hash_dir)) self.assertFalse(os.path.exists(temp_file.name)) self.assertFalse(os.path.exists(temp_hash_dir2)) self.assertFalse(os.path.exists(temp_file2.name)) self.assertEqual([(('removes.some_device',), {})] * 2, replicator.logger.log_dict['increment']) self.assertEqual(2, replicator.stats['remove']) finally: rmtree(temp_dir) def test_extract_device(self): replicator = TestReplicator({'devices': '/some/root'}) self.assertEqual('some_device', replicator.extract_device( '/some/root/some_device/deeper/and/deeper')) self.assertEqual('UNKNOWN', replicator.extract_device( '/some/foo/some_device/deeper/and/deeper')) # def test_dispatch(self): # rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) # no_op = lambda *args, **kwargs: True # self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ('op',) # ).status_int, 400) # rpc.mount_check = True # self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ['op',] # ).status_int, 507) # rpc.mount_check = False # rpc.rsync_then_merge = lambda drive, db_file, # args: self.assertEquals(args, ['test1']) # rpc.complete_rsync = lambda drive, db_file, # args: self.assertEquals(args, ['test2']) # rpc.dispatch(('drv', 'part', 'hash'), ['rsync_then_merge','test1']) # rpc.dispatch(('drv', 'part', 'hash'), ['complete_rsync','test2']) # rpc.dispatch(('drv', 'part', 'hash'), ['other_op',]) def test_dispatch_no_arg_pop(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) response = rpc.dispatch(('a',), 'arg') self.assertEquals('Invalid object type', response.body) self.assertEquals(400, response.status_int) def test_dispatch_drive_not_mounted(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, True) def mock_ismount(path): self.assertEquals('/drive', path) return False self._patch(patch.object, db_replicator, 'ismount', mock_ismount) response = rpc.dispatch(('drive', 'part', 'hash'), ['method']) self.assertEquals('507 drive is not mounted', response.status) self.assertEquals(507, response.status_int) def test_dispatch_unexpected_operation_db_does_not_exist(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) def mock_mkdirs(path): self.assertEquals('/drive/tmp', path) self._patch(patch.object, db_replicator, 'mkdirs', mock_mkdirs) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = False response = rpc.dispatch(('drive', 'part', 'hash'), ['unexpected']) self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_dispatch_operation_unexpected(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) self._patch(patch.object, db_replicator, 'mkdirs', lambda *args: True) def unexpected_method(broker, args): self.assertEquals(FakeBroker, broker.__class__) self.assertEqual(['arg1', 'arg2'], args) return 'unexpected-called' rpc.unexpected = unexpected_method with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = True response = rpc.dispatch(('drive', 'part', 'hash'), ['unexpected', 'arg1', 'arg2']) mock_os.path.exists.assert_called_with('/part/ash/hash/hash.db') self.assertEquals('unexpected-called', response) def test_dispatch_operation_rsync_then_merge(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) self._patch(patch.object, db_replicator, 'renamer', lambda *args: True) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = True response = rpc.dispatch(('drive', 'part', 'hash'), ['rsync_then_merge', 'arg1', 'arg2']) expected_calls = [call('/part/ash/hash/hash.db'), call('/drive/tmp/arg1')] self.assertEquals(mock_os.path.exists.call_args_list, expected_calls) self.assertEquals('204 No Content', response.status) self.assertEquals(204, response.status_int) def test_dispatch_operation_complete_rsync(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) self._patch(patch.object, db_replicator, 'renamer', lambda *args: True) with patch('swift.common.db_replicator.os', new=mock.MagicMock( wraps=os)) as mock_os: mock_os.path.exists.side_effect = [False, True] response = rpc.dispatch(('drive', 'part', 'hash'), ['complete_rsync', 'arg1', 'arg2']) expected_calls = [call('/part/ash/hash/hash.db'), call('/drive/tmp/arg1')] self.assertEquals(mock_os.path.exists.call_args_list, expected_calls) self.assertEquals('204 No Content', response.status) self.assertEquals(204, response.status_int) def test_rsync_then_merge_db_does_not_exist(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = False response = rpc.rsync_then_merge('drive', '/data/db.db', ('arg1', 'arg2')) mock_os.path.exists.assert_called_with('/data/db.db') self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_rsync_then_merge_old_does_not_exist(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.side_effect = [True, False] response = rpc.rsync_then_merge('drive', '/data/db.db', ('arg1', 'arg2')) expected_calls = [call('/data/db.db'), call('/drive/tmp/arg1')] self.assertEquals(mock_os.path.exists.call_args_list, expected_calls) self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_rsync_then_merge_with_objects(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) def mock_renamer(old, new): self.assertEquals('/drive/tmp/arg1', old) self.assertEquals('/data/db.db', new) self._patch(patch.object, db_replicator, 'renamer', mock_renamer) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = True response = rpc.rsync_then_merge('drive', '/data/db.db', ['arg1', 'arg2']) self.assertEquals('204 No Content', response.status) self.assertEquals(204, response.status_int) def test_complete_rsync_db_does_not_exist(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = True response = rpc.complete_rsync('drive', '/data/db.db', ['arg1', 'arg2']) mock_os.path.exists.assert_called_with('/data/db.db') self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_complete_rsync_old_file_does_not_exist(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.return_value = False response = rpc.complete_rsync('drive', '/data/db.db', ['arg1', 'arg2']) expected_calls = [call('/data/db.db'), call('/drive/tmp/arg1')] self.assertEquals(expected_calls, mock_os.path.exists.call_args_list) self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_complete_rsync_rename(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) def mock_exists(path): if path == '/data/db.db': return False self.assertEquals('/drive/tmp/arg1', path) return True def mock_renamer(old, new): self.assertEquals('/drive/tmp/arg1', old) self.assertEquals('/data/db.db', new) self._patch(patch.object, db_replicator, 'renamer', mock_renamer) with patch('swift.common.db_replicator.os', new=mock.MagicMock(wraps=os)) as mock_os: mock_os.path.exists.side_effect = [False, True] response = rpc.complete_rsync('drive', '/data/db.db', ['arg1', 'arg2']) self.assertEquals('204 No Content', response.status) self.assertEquals(204, response.status_int) def test_replicator_sync_with_broker_replication_missing_table(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) broker = FakeBroker() broker.get_repl_missing_table = True def mock_quarantine_db(object_file, server_type): self.assertEquals(broker.db_file, object_file) self.assertEquals(broker.db_type, server_type) self._patch(patch.object, db_replicator, 'quarantine_db', mock_quarantine_db) response = rpc.sync(broker, ('remote_sync', 'hash_', 'id_', 'created_at', 'put_timestamp', 'delete_timestamp', 'metadata')) self.assertEquals('404 Not Found', response.status) self.assertEquals(404, response.status_int) def test_replicator_sync(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) broker = FakeBroker() response = rpc.sync(broker, (broker.get_sync() + 1, 12345, 'id_', 'created_at', 'put_timestamp', 'delete_timestamp', '{"meta1": "data1", "meta2": "data2"}')) self.assertEquals({'meta1': 'data1', 'meta2': 'data2'}, broker.metadata) self.assertEquals('created_at', broker.created_at) self.assertEquals('put_timestamp', broker.put_timestamp) self.assertEquals('delete_timestamp', broker.delete_timestamp) self.assertEquals('200 OK', response.status) self.assertEquals(200, response.status_int) def test_rsync_then_merge(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) rpc.rsync_then_merge('sda1', '/srv/swift/blah', ('a', 'b')) def test_merge_items(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) fake_broker = FakeBroker() args = ('a', 'b') rpc.merge_items(fake_broker, args) self.assertEquals(fake_broker.args, args) def test_merge_syncs(self): rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) fake_broker = FakeBroker() args = ('a', 'b') rpc.merge_syncs(fake_broker, args) self.assertEquals(fake_broker.args, (args[0],)) def test_complete_rsync_with_bad_input(self): drive = '/some/root' db_file = __file__ args = ['old_file'] rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) resp = rpc.complete_rsync(drive, db_file, args) self.assertTrue(isinstance(resp, HTTPException)) self.assertEquals(404, resp.status_int) resp = rpc.complete_rsync(drive, 'new_db_file', args) self.assertTrue(isinstance(resp, HTTPException)) self.assertEquals(404, resp.status_int) def test_complete_rsync(self): drive = mkdtemp() args = ['old_file'] rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False) os.mkdir('%s/tmp' % drive) old_file = '%s/tmp/old_file' % drive new_file = '%s/new_db_file' % drive try: fp = open(old_file, 'w') fp.write('void') fp.close resp = rpc.complete_rsync(drive, new_file, args) self.assertEquals(204, resp.status_int) finally: rmtree(drive) def test_roundrobin_datadirs(self): listdir_calls = [] isdir_calls = [] exists_calls = [] shuffle_calls = [] def _listdir(path): listdir_calls.append(path) if not path.startswith('/srv/node/sda/containers') and \ not path.startswith('/srv/node/sdb/containers'): return [] path = path[len('/srv/node/sdx/containers'):] if path == '': return ['123', '456', '789'] # 456 will pretend to be a file elif path == '/123': return ['abc', 'def.db'] # def.db will pretend to be a file elif path == '/123/abc': # 11111111111111111111111111111abc will pretend to be a file return ['00000000000000000000000000000abc', '11111111111111111111111111111abc'] elif path == '/123/abc/00000000000000000000000000000abc': return ['00000000000000000000000000000abc.db', # This other.db isn't in the right place, so should be # ignored later. '000000000000000000000000000other.db', 'weird1'] # weird1 will pretend to be a dir, if asked elif path == '/789': return ['ghi', 'jkl'] # jkl will pretend to be a file elif path == '/789/ghi': # 33333333333333333333333333333ghi will pretend to be a file return ['22222222222222222222222222222ghi', '33333333333333333333333333333ghi'] elif path == '/789/ghi/22222222222222222222222222222ghi': return ['22222222222222222222222222222ghi.db', 'weird2'] # weird2 will pretend to be a dir, if asked return [] def _isdir(path): isdir_calls.append(path) if not path.startswith('/srv/node/sda/containers') and \ not path.startswith('/srv/node/sdb/containers'): return False path = path[len('/srv/node/sdx/containers'):] if path in ('/123', '/123/abc', '/123/abc/00000000000000000000000000000abc', '/123/abc/00000000000000000000000000000abc/weird1', '/789', '/789/ghi', '/789/ghi/22222222222222222222222222222ghi', '/789/ghi/22222222222222222222222222222ghi/weird2'): return True return False def _exists(arg): exists_calls.append(arg) return True def _shuffle(arg): shuffle_calls.append(arg) orig_listdir = db_replicator.os.listdir orig_isdir = db_replicator.os.path.isdir orig_exists = db_replicator.os.path.exists orig_shuffle = db_replicator.random.shuffle try: db_replicator.os.listdir = _listdir db_replicator.os.path.isdir = _isdir db_replicator.os.path.exists = _exists db_replicator.random.shuffle = _shuffle datadirs = [('/srv/node/sda/containers', 1), ('/srv/node/sdb/containers', 2)] results = list(db_replicator.roundrobin_datadirs(datadirs)) # The results show that the .db files are returned, the devices # interleaved. self.assertEquals(results, [ ('123', '/srv/node/sda/containers/123/abc/' '00000000000000000000000000000abc/' '00000000000000000000000000000abc.db', 1), ('123', '/srv/node/sdb/containers/123/abc/' '00000000000000000000000000000abc/' '00000000000000000000000000000abc.db', 2), ('789', '/srv/node/sda/containers/789/ghi/' '22222222222222222222222222222ghi/' '22222222222222222222222222222ghi.db', 1), ('789', '/srv/node/sdb/containers/789/ghi/' '22222222222222222222222222222ghi/' '22222222222222222222222222222ghi.db', 2)]) # The listdir calls show that we only listdir the dirs self.assertEquals(listdir_calls, [ '/srv/node/sda/containers', '/srv/node/sda/containers/123', '/srv/node/sda/containers/123/abc', '/srv/node/sdb/containers', '/srv/node/sdb/containers/123', '/srv/node/sdb/containers/123/abc', '/srv/node/sda/containers/789', '/srv/node/sda/containers/789/ghi', '/srv/node/sdb/containers/789', '/srv/node/sdb/containers/789/ghi']) # The isdir calls show that we did ask about the things pretending # to be files at various levels. self.assertEquals(isdir_calls, [ '/srv/node/sda/containers/123', '/srv/node/sda/containers/123/abc', ('/srv/node/sda/containers/123/abc/' '00000000000000000000000000000abc'), '/srv/node/sdb/containers/123', '/srv/node/sdb/containers/123/abc', ('/srv/node/sdb/containers/123/abc/' '00000000000000000000000000000abc'), ('/srv/node/sda/containers/123/abc/' '11111111111111111111111111111abc'), '/srv/node/sda/containers/123/def.db', '/srv/node/sda/containers/456', '/srv/node/sda/containers/789', '/srv/node/sda/containers/789/ghi', ('/srv/node/sda/containers/789/ghi/' '22222222222222222222222222222ghi'), ('/srv/node/sdb/containers/123/abc/' '11111111111111111111111111111abc'), '/srv/node/sdb/containers/123/def.db', '/srv/node/sdb/containers/456', '/srv/node/sdb/containers/789', '/srv/node/sdb/containers/789/ghi', ('/srv/node/sdb/containers/789/ghi/' '22222222222222222222222222222ghi'), ('/srv/node/sda/containers/789/ghi/' '33333333333333333333333333333ghi'), '/srv/node/sda/containers/789/jkl', ('/srv/node/sdb/containers/789/ghi/' '33333333333333333333333333333ghi'), '/srv/node/sdb/containers/789/jkl']) # The exists calls are the .db files we looked for as we walked the # structure. self.assertEquals(exists_calls, [ ('/srv/node/sda/containers/123/abc/' '00000000000000000000000000000abc/' '00000000000000000000000000000abc.db'), ('/srv/node/sdb/containers/123/abc/' '00000000000000000000000000000abc/' '00000000000000000000000000000abc.db'), ('/srv/node/sda/containers/789/ghi/' '22222222222222222222222222222ghi/' '22222222222222222222222222222ghi.db'), ('/srv/node/sdb/containers/789/ghi/' '22222222222222222222222222222ghi/' '22222222222222222222222222222ghi.db')]) # Shows that we called shuffle twice, once for each device. self.assertEquals( shuffle_calls, [['123', '456', '789'], ['123', '456', '789']]) finally: db_replicator.os.listdir = orig_listdir db_replicator.os.path.isdir = orig_isdir db_replicator.os.path.exists = orig_exists db_replicator.random.shuffle = orig_shuffle @mock.patch("swift.common.db_replicator.ReplConnection", mock.Mock()) def test_http_connect(self): node = "node" partition = "partition" db_file = __file__ replicator = TestReplicator({}) replicator._http_connect(node, partition, db_file) db_replicator.ReplConnection.assert_has_calls( mock.call(node, partition, os.path.basename(db_file).split('.', 1)[0], replicator.logger)) class TestReplToNode(unittest.TestCase): def setUp(self): db_replicator.ring = FakeRing() self.delete_db_calls = [] self.broker = FakeBroker() self.replicator = TestReplicator({}) self.fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000} self.fake_info = {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'b', 'created_at': 100, 'put_timestamp': 0, 'delete_timestamp': 0, 'count': 0, 'metadata': { 'Test': ('Value', normalize_timestamp(1))}} self.replicator.logger = mock.Mock() self.replicator._rsync_db = mock.Mock(return_value=True) self.replicator._usync_db = mock.Mock(return_value=True) self.http = ReplHttp('{"id": 3, "point": -1}') self.replicator._http_connect = lambda *args: self.http def test_repl_to_node_usync_success(self): rinfo = {"id": 3, "point": -1, "max_row": 5, "hash": "c"} self.http = ReplHttp(simplejson.dumps(rinfo)) local_sync = self.broker.get_sync() self.assertEquals(self.replicator._repl_to_node( self.fake_node, self.broker, '0', self.fake_info), True) self.replicator._usync_db.assert_has_calls([ mock.call(max(rinfo['point'], local_sync), self.broker, self.http, rinfo['id'], self.fake_info['id']) ]) def test_repl_to_node_rsync_success(self): rinfo = {"id": 3, "point": -1, "max_row": 4, "hash": "c"} self.http = ReplHttp(simplejson.dumps(rinfo)) self.broker.get_sync() self.assertEquals(self.replicator._repl_to_node( self.fake_node, self.broker, '0', self.fake_info), True) self.replicator.logger.increment.assert_has_calls([ mock.call.increment('remote_merges') ]) self.replicator._rsync_db.assert_has_calls([ mock.call(self.broker, self.fake_node, self.http, self.fake_info['id'], replicate_method='rsync_then_merge', replicate_timeout=(self.fake_info['count'] / 2000)) ]) def test_repl_to_node_already_in_sync(self): rinfo = {"id": 3, "point": -1, "max_row": 10, "hash": "b"} self.http = ReplHttp(simplejson.dumps(rinfo)) self.broker.get_sync() self.assertEquals(self.replicator._repl_to_node( self.fake_node, self.broker, '0', self.fake_info), True) self.assertEquals(self.replicator._rsync_db.call_count, 0) self.assertEquals(self.replicator._usync_db.call_count, 0) def test_repl_to_node_not_found(self): self.http = ReplHttp('{"id": 3, "point": -1}', set_status=404) self.assertEquals(self.replicator._repl_to_node( self.fake_node, self.broker, '0', self.fake_info), True) self.replicator.logger.increment.assert_has_calls([ mock.call.increment('rsyncs') ]) self.replicator._rsync_db.assert_has_calls([ mock.call(self.broker, self.fake_node, self.http, self.fake_info['id']) ]) def test_repl_to_node_drive_not_mounted(self): self.http = ReplHttp('{"id": 3, "point": -1}', set_status=507) self.assertRaises(DriveNotMounted, self.replicator._repl_to_node, self.fake_node, FakeBroker(), '0', self.fake_info) def test_repl_to_node_300_status(self): self.http = ReplHttp('{"id": 3, "point": -1}', set_status=300) self.assertEquals(self.replicator._repl_to_node( self.fake_node, FakeBroker(), '0', self.fake_info), None) def test_repl_to_node_http_connect_fails(self): self.replicator._http_connect = lambda *args: None self.assertEquals(self.replicator._repl_to_node( self.fake_node, FakeBroker(), '0', self.fake_info), False) def test_repl_to_node_not_response(self): self.http = mock.Mock(replicate=mock.Mock(return_value=None)) self.assertEquals(self.replicator._repl_to_node( self.fake_node, FakeBroker(), '0', self.fake_info), False) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/ring/0000775000175400017540000000000012323703665020263 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/ring/test_builder.py0000664000175400017540000012353512323703611023322 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mock import operator import os import unittest import cPickle as pickle from collections import defaultdict from tempfile import mkdtemp from shutil import rmtree from swift.common import exceptions from swift.common import ring from swift.common.ring.builder import MAX_BALANCE class TestRingBuilder(unittest.TestCase): def setUp(self): self.testdir = mkdtemp() def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_init(self): rb = ring.RingBuilder(8, 3, 1) self.assertEquals(rb.part_power, 8) self.assertEquals(rb.replicas, 3) self.assertEquals(rb.min_part_hours, 1) self.assertEquals(rb.parts, 2 ** 8) self.assertEquals(rb.devs, []) self.assertEquals(rb.devs_changed, False) self.assertEquals(rb.version, 0) def test_overlarge_part_powers(self): ring.RingBuilder(32, 3, 1) # passes by not crashing self.assertRaises(ValueError, ring.RingBuilder, 33, 3, 1) def test_insufficient_replicas(self): ring.RingBuilder(8, 1.0, 1) # passes by not crashing self.assertRaises(ValueError, ring.RingBuilder, 8, 0.999, 1) def test_negative_min_part_hours(self): ring.RingBuilder(8, 3, 0) # passes by not crashing self.assertRaises(ValueError, ring.RingBuilder, 8, 3, -1) def test_get_ring(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.remove_dev(1) rb.rebalance() r = rb.get_ring() self.assert_(isinstance(r, ring.RingData)) r2 = rb.get_ring() self.assert_(r is r2) rb.rebalance() r3 = rb.get_ring() self.assert_(r3 is not r2) r4 = rb.get_ring() self.assert_(r3 is r4) def test_rebalance_with_seed(self): devs = [(0, 10000), (1, 10001), (2, 10002), (1, 10003)] ring_builders = [] for n in range(3): rb = ring.RingBuilder(8, 3, 1) for idx, (zone, port) in enumerate(devs): rb.add_dev({'id': idx, 'region': 0, 'zone': zone, 'weight': 1, 'ip': '127.0.0.1', 'port': port, 'device': 'sda1'}) ring_builders.append(rb) rb0 = ring_builders[0] rb1 = ring_builders[1] rb2 = ring_builders[2] r0 = rb0.get_ring() self.assertTrue(rb0.get_ring() is r0) rb0.rebalance() # NO SEED rb1.rebalance(seed=10) rb2.rebalance(seed=10) r1 = rb1.get_ring() r2 = rb2.get_ring() self.assertFalse(rb0.get_ring() is r0) self.assertNotEquals(r0.to_dict(), r1.to_dict()) self.assertEquals(r1.to_dict(), r2.to_dict()) def test_rebalance_part_on_deleted_other_part_on_drained(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.add_dev({'id': 3, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.add_dev({'id': 4, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.add_dev({'id': 5, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10005, 'device': 'sda1'}) rb.rebalance(seed=1) # We want a partition where 1 replica is on a removed device, 1 # replica is on a 0-weight device, and 1 on a normal device. To # guarantee we have one, we see where partition 123 is, then # manipulate its devices accordingly. zero_weight_dev_id = rb._replica2part2dev[1][123] delete_dev_id = rb._replica2part2dev[2][123] rb.set_dev_weight(zero_weight_dev_id, 0.0) rb.remove_dev(delete_dev_id) rb.rebalance() def test_set_replicas(self): rb = ring.RingBuilder(8, 3.2, 1) rb.devs_changed = False rb.set_replicas(3.25) self.assertTrue(rb.devs_changed) rb.devs_changed = False rb.set_replicas(3.2500001) self.assertFalse(rb.devs_changed) def test_add_dev(self): rb = ring.RingBuilder(8, 3, 1) dev = {'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000} dev_id = rb.add_dev(dev) self.assertRaises(exceptions.DuplicateDeviceError, rb.add_dev, dev) self.assertEqual(dev_id, 0) rb = ring.RingBuilder(8, 3, 1) # test add new dev with no id dev_id = rb.add_dev({'zone': 0, 'region': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) self.assertEquals(rb.devs[0]['id'], 0) self.assertEqual(dev_id, 0) #test add another dev with no id dev_id = rb.add_dev({'zone': 3, 'region': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 6000}) self.assertEquals(rb.devs[1]['id'], 1) self.assertEqual(dev_id, 1) def test_set_dev_weight(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 0.5, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 128, 1: 128, 2: 256, 3: 256}) rb.set_dev_weight(0, 0.75) rb.set_dev_weight(1, 0.25) rb.pretend_min_part_hours_passed() rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 192, 1: 64, 2: 256, 3: 256}) def test_remove_dev(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 192, 1: 192, 2: 192, 3: 192}) rb.remove_dev(1) rb.pretend_min_part_hours_passed() rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 256, 2: 256, 3: 256}) def test_remove_a_lot(self): rb = ring.RingBuilder(3, 3, 1) rb.add_dev({'id': 0, 'device': 'd0', 'ip': '10.0.0.1', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1}) rb.add_dev({'id': 1, 'device': 'd1', 'ip': '10.0.0.2', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2}) rb.add_dev({'id': 2, 'device': 'd2', 'ip': '10.0.0.3', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3}) rb.add_dev({'id': 3, 'device': 'd3', 'ip': '10.0.0.1', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 1}) rb.add_dev({'id': 4, 'device': 'd4', 'ip': '10.0.0.2', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 2}) rb.add_dev({'id': 5, 'device': 'd5', 'ip': '10.0.0.3', 'port': 6002, 'weight': 1000.0, 'region': 0, 'zone': 3}) rb.rebalance() rb.validate() # this has to put more than 1/3 of the partitions in the # cluster on removed devices in order to ensure that at least # one partition has multiple replicas that need to move. # # (for an N-replica ring, it's more than 1/N of the # partitions, of course) rb.remove_dev(3) rb.remove_dev(4) rb.remove_dev(5) rb.rebalance() rb.validate() def test_shuffled_gather(self): if self._shuffled_gather_helper() and \ self._shuffled_gather_helper(): raise AssertionError('It is highly likely the ring is no ' 'longer shuffling the set of partitions ' 'to reassign on a rebalance.') def _shuffled_gather_helper(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() parts = rb._gather_reassign_parts() max_run = 0 run = 0 last_part = 0 for part, _ in parts: if part > last_part: run += 1 else: if run > max_run: max_run = run run = 0 last_part = part if run > max_run: max_run = run return max_run > len(parts) / 2 def test_initial_balance(self): # 2 boxes, 2 drives each in zone 1 # 1 box, 2 drives in zone 2 # # This is balanceable, but there used to be some nondeterminism in # rebalance() that would sometimes give you an imbalanced ring. rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'region': 1, 'zone': 1, 'weight': 4000.0, 'ip': '10.1.1.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'region': 1, 'zone': 1, 'weight': 4000.0, 'ip': '10.1.1.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'region': 1, 'zone': 1, 'weight': 4000.0, 'ip': '10.1.1.2', 'port': 10000, 'device': 'sda'}) rb.add_dev({'region': 1, 'zone': 1, 'weight': 4000.0, 'ip': '10.1.1.2', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'region': 1, 'zone': 2, 'weight': 4000.0, 'ip': '10.1.1.3', 'port': 10000, 'device': 'sda'}) rb.add_dev({'region': 1, 'zone': 2, 'weight': 4000.0, 'ip': '10.1.1.3', 'port': 10000, 'device': 'sdb'}) _, balance = rb.rebalance(seed=2) # maybe not *perfect*, but should be close self.assert_(balance <= 1) def test_multitier_partial(self): # Multitier test, nothing full rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'id': 2, 'region': 2, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.add_dev({'id': 3, 'region': 3, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['region'][dev['region']] += 1 counts['zone'][dev['zone']] += 1 if any(c > 1 for c in counts['region'].values()): raise AssertionError( "Partition %d not evenly region-distributed (got %r)" % (part, counts['region'])) if any(c > 1 for c in counts['zone'].values()): raise AssertionError( "Partition %d not evenly zone-distributed (got %r)" % (part, counts['zone'])) # Multitier test, zones full, nodes not full rb = ring.RingBuilder(8, 6, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) rb.add_dev({'id': 5, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) rb.add_dev({'id': 6, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdg'}) rb.add_dev({'id': 7, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdh'}) rb.add_dev({'id': 8, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sdi'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 counts['dev_id'][dev['id']] += 1 if counts['zone'] != {0: 2, 1: 2, 2: 2}: raise AssertionError( "Partition %d not evenly distributed (got %r)" % (part, counts['zone'])) for dev_id, replica_count in counts['dev_id'].iteritems(): if replica_count > 1: raise AssertionError( "Partition %d is on device %d more than once (%r)" % (part, dev_id, counts['dev_id'])) def test_multitier_full(self): # Multitier test, #replicas == #devs rb = ring.RingBuilder(8, 6, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 counts['dev_id'][dev['id']] += 1 if counts['zone'] != {0: 2, 1: 2, 2: 2}: raise AssertionError( "Partition %d not evenly distributed (got %r)" % (part, counts['zone'])) for dev_id, replica_count in counts['dev_id'].iteritems(): if replica_count != 1: raise AssertionError( "Partition %d is on device %d %d times, not 1 (%r)" % (part, dev_id, replica_count, counts['dev_id'])) def test_multitier_overfull(self): # Multitier test, #replicas > #devs + 2 (to prove even distribution) rb = ring.RingBuilder(8, 8, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdd'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sde'}) rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdf'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): counts = defaultdict(lambda: defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 counts['dev_id'][dev['id']] += 1 self.assertEquals(8, sum(counts['zone'].values())) for zone, replica_count in counts['zone'].iteritems(): if replica_count not in (2, 3): raise AssertionError( "Partition %d not evenly distributed (got %r)" % (part, counts['zone'])) for dev_id, replica_count in counts['dev_id'].iteritems(): if replica_count not in (1, 2): raise AssertionError( "Partition %d is on device %d %d times, " "not 1 or 2 (%r)" % (part, dev_id, replica_count, counts['dev_id'])) def test_multitier_expansion_more_devices(self): rb = ring.RingBuilder(8, 6, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc'}) rb.rebalance() rb.validate() rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde'}) rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf'}) for _ in xrange(5): rb.pretend_min_part_hours_passed() rb.rebalance() rb.validate() for part in xrange(rb.parts): counts = dict(zone=defaultdict(int), dev_id=defaultdict(int)) for replica in xrange(rb.replicas): dev = rb.devs[rb._replica2part2dev[replica][part]] counts['zone'][dev['zone']] += 1 counts['dev_id'][dev['id']] += 1 self.assertEquals({0: 2, 1: 2, 2: 2}, dict(counts['zone'])) self.assertEquals({0: 1, 1: 1, 2: 1, 3: 1, 4: 1, 5: 1}, dict(counts['dev_id'])) def test_multitier_part_moves_with_0_min_part_hours(self): rb = ring.RingBuilder(8, 3, 0) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.rebalance() rb.validate() # min_part_hours is 0, so we're clear to move 2 replicas to # new devs rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): devs = set() for replica in xrange(rb.replicas): devs.add(rb._replica2part2dev[replica][part]) if len(devs) != 3: raise AssertionError( "Partition %d not on 3 devs (got %r)" % (part, devs)) def test_multitier_part_moves_with_positive_min_part_hours(self): rb = ring.RingBuilder(8, 3, 99) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.rebalance() rb.validate() # min_part_hours is >0, so we'll only be able to move 1 # replica to a new home rb.add_dev({'id': 1, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdc1'}) rb.pretend_min_part_hours_passed() rb.rebalance() rb.validate() for part in xrange(rb.parts): devs = set() for replica in xrange(rb.replicas): devs.add(rb._replica2part2dev[replica][part]) if len(devs) != 2: raise AssertionError( "Partition %d not on 2 devs (got %r)" % (part, devs)) def test_multitier_dont_move_too_many_replicas(self): rb = ring.RingBuilder(8, 3, 0) # there'll be at least one replica in z0 and z1 rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdb1'}) rb.rebalance() rb.validate() # only 1 replica should move rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdd1'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sde1'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 4, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sdf1'}) rb.rebalance() rb.validate() for part in xrange(rb.parts): zones = set() for replica in xrange(rb.replicas): zones.add(rb.devs[rb._replica2part2dev[replica][part]]['zone']) if len(zones) != 3: raise AssertionError( "Partition %d not in 3 zones (got %r)" % (part, zones)) if 0 not in zones or 1 not in zones: raise AssertionError( "Partition %d not in zones 0 and 1 (got %r)" % (part, zones)) def test_rerebalance(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 256, 1: 256, 2: 256}) rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 192, 1: 192, 2: 192, 3: 192}) rb.set_dev_weight(3, 100) rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts[3], 256) def test_add_rebalance_add_rebalance_delete_rebalance(self): # Test for https://bugs.launchpad.net/swift/+bug/845952 # min_part of 0 to allow for rapid rebalancing rb = ring.RingBuilder(8, 3, 0) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.rebalance() rb.add_dev({'id': 3, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) rb.add_dev({'id': 4, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.add_dev({'id': 5, 'region': 0, 'zone': 2, 'weight': 1, 'ip': '127.0.0.1', 'port': 10005, 'device': 'sda1'}) rb.rebalance() rb.remove_dev(1) rb.rebalance() def test_set_replicas_increase(self): rb = ring.RingBuilder(8, 2, 0) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() rb.validate() rb.replicas = 2.1 rb.rebalance() rb.validate() self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], [256, 256, 25]) rb.replicas = 2.2 rb.rebalance() rb.validate() self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], [256, 256, 51]) def test_set_replicas_decrease(self): rb = ring.RingBuilder(4, 5, 0) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() rb.validate() rb.replicas = 4.9 rb.rebalance() rb.validate() self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], [16, 16, 16, 16, 14]) # cross a couple of integer thresholds (4 and 3) rb.replicas = 2.5 rb.rebalance() rb.validate() self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], [16, 16, 8]) def test_fractional_replicas_rebalance(self): rb = ring.RingBuilder(8, 2.5, 0) rb.add_dev({'id': 0, 'region': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() # passes by not crashing rb.validate() # also passes by not crashing self.assertEqual([len(p2d) for p2d in rb._replica2part2dev], [256, 256, 128]) def test_load(self): rb = ring.RingBuilder(8, 3, 1) devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 2, 'region': 0, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, {'id': 3, 'region': 0, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}] for d in devs: rb.add_dev(d) rb.rebalance() real_pickle = pickle.load try: #test a legit builder fake_pickle = mock.Mock(return_value=rb) fake_open = mock.Mock(return_value=None) pickle.load = fake_pickle builder = ring.RingBuilder.load('fake.builder', open=fake_open) self.assertEquals(fake_pickle.call_count, 1) fake_open.assert_has_calls([mock.call('fake.builder', 'rb')]) self.assertEquals(builder, rb) fake_pickle.reset_mock() fake_open.reset_mock() #test old style builder fake_pickle.return_value = rb.to_dict() pickle.load = fake_pickle builder = ring.RingBuilder.load('fake.builder', open=fake_open) fake_open.assert_has_calls([mock.call('fake.builder', 'rb')]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() fake_open.reset_mock() #test old devs but no meta no_meta_builder = rb for dev in no_meta_builder.devs: del(dev['meta']) fake_pickle.return_value = no_meta_builder pickle.load = fake_pickle builder = ring.RingBuilder.load('fake.builder', open=fake_open) fake_open.assert_has_calls([mock.call('fake.builder', 'rb')]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() finally: pickle.load = real_pickle def test_save_load(self): rb = ring.RingBuilder(8, 3, 1) devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', 'port': 10000, 'replication_ip': '127.0.0.0', 'replication_port': 10000, 'device': 'sda1', 'meta': 'meta0'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'replication_ip': '127.0.0.1', 'replication_port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 2, 'region': 0, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'port': 10002, 'replication_ip': '127.0.0.2', 'replication_port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, {'id': 3, 'region': 0, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', 'port': 10003, 'replication_ip': '127.0.0.3', 'replication_port': 10003, 'device': 'sdd1', 'meta': ''}] for d in devs: rb.add_dev(d) rb.rebalance() builder_file = os.path.join(self.testdir, 'test_save.builder') rb.save(builder_file) loaded_rb = ring.RingBuilder.load(builder_file) self.maxDiff = None self.assertEquals(loaded_rb.to_dict(), rb.to_dict()) @mock.patch('__builtin__.open', autospec=True) @mock.patch('swift.common.ring.builder.pickle.dump', autospec=True) def test_save(self, mock_pickle_dump, mock_open): mock_open.return_value = mock_fh = mock.MagicMock() rb = ring.RingBuilder(8, 3, 1) devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 2, 'region': 0, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, {'id': 3, 'region': 0, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}] for d in devs: rb.add_dev(d) rb.rebalance() rb.save('some.builder') mock_open.assert_called_once_with('some.builder', 'wb') mock_pickle_dump.assert_called_once_with(rb.to_dict(), mock_fh.__enter__(), protocol=2) def test_search_devs(self): rb = ring.RingBuilder(8, 3, 1) devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 2, 'region': 1, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, {'id': 3, 'region': 1, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1', 'meta': 'meta3'}, {'id': 4, 'region': 2, 'zone': 4, 'weight': 1, 'ip': '127.0.0.4', 'port': 10004, 'device': 'sde1', 'meta': 'meta4', 'replication_ip': '127.0.0.10', 'replication_port': 20000}, {'id': 5, 'region': 2, 'zone': 5, 'weight': 2, 'ip': '127.0.0.5', 'port': 10005, 'device': 'sdf1', 'meta': 'meta5', 'replication_ip': '127.0.0.11', 'replication_port': 20001}, {'id': 6, 'region': 2, 'zone': 6, 'weight': 2, 'ip': '127.0.0.6', 'port': 10006, 'device': 'sdg1', 'meta': 'meta6', 'replication_ip': '127.0.0.12', 'replication_port': 20002}] for d in devs: rb.add_dev(d) rb.rebalance() res = rb.search_devs({'region': 0}) self.assertEquals(res, [devs[0], devs[1]]) res = rb.search_devs({'region': 1}) self.assertEquals(res, [devs[2], devs[3]]) res = rb.search_devs({'region': 1, 'zone': 2}) self.assertEquals(res, [devs[2]]) res = rb.search_devs({'id': 1}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'zone': 1}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'ip': '127.0.0.1'}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'ip': '127.0.0.1', 'port': 10001}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'port': 10001}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'replication_ip': '127.0.0.10'}) self.assertEquals(res, [devs[4]]) res = rb.search_devs({'replication_ip': '127.0.0.10', 'replication_port': 20000}) self.assertEquals(res, [devs[4]]) res = rb.search_devs({'replication_port': 20000}) self.assertEquals(res, [devs[4]]) res = rb.search_devs({'device': 'sdb1'}) self.assertEquals(res, [devs[1]]) res = rb.search_devs({'meta': 'meta1'}) self.assertEquals(res, [devs[1]]) def test_validate(self): rb = ring.RingBuilder(8, 3, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.add_dev({'id': 2, 'region': 0, 'zone': 2, 'weight': 2, 'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'}) rb.add_dev({'id': 3, 'region': 0, 'zone': 3, 'weight': 2, 'ip': '127.0.0.1', 'port': 10003, 'device': 'sda1'}) # Degenerate case: devices added but not rebalanced yet self.assertRaises(exceptions.RingValidationError, rb.validate) rb.rebalance() r = rb.get_ring() counts = {} for part2dev_id in r._replica2part2dev_id: for dev_id in part2dev_id: counts[dev_id] = counts.get(dev_id, 0) + 1 self.assertEquals(counts, {0: 128, 1: 128, 2: 256, 3: 256}) dev_usage, worst = rb.validate() self.assert_(dev_usage is None) self.assert_(worst is None) dev_usage, worst = rb.validate(stats=True) self.assertEquals(list(dev_usage), [128, 128, 256, 256]) self.assertEquals(int(worst), 0) rb.set_dev_weight(2, 0) rb.rebalance() self.assertEquals(rb.validate(stats=True)[1], MAX_BALANCE) # Test not all partitions doubly accounted for rb.devs[1]['parts'] -= 1 self.assertRaises(exceptions.RingValidationError, rb.validate) rb.devs[1]['parts'] += 1 # Test non-numeric port rb.devs[1]['port'] = '10001' self.assertRaises(exceptions.RingValidationError, rb.validate) rb.devs[1]['port'] = 10001 # Test partition on nonexistent device rb.pretend_min_part_hours_passed() orig_dev_id = rb._replica2part2dev[0][0] rb._replica2part2dev[0][0] = len(rb.devs) self.assertRaises(exceptions.RingValidationError, rb.validate) rb._replica2part2dev[0][0] = orig_dev_id # Tests that validate can handle 'holes' in .devs rb.remove_dev(2) rb.pretend_min_part_hours_passed() rb.rebalance() rb.validate(stats=True) # Test partition assigned to a hole if rb.devs[2]: rb.remove_dev(2) rb.pretend_min_part_hours_passed() orig_dev_id = rb._replica2part2dev[0][0] rb._replica2part2dev[0][0] = 2 self.assertRaises(exceptions.RingValidationError, rb.validate) rb._replica2part2dev[0][0] = orig_dev_id # Validate that zero weight devices with no partitions don't count on # the 'worst' value. self.assertNotEquals(rb.validate(stats=True)[1], MAX_BALANCE) rb.add_dev({'id': 4, 'region': 0, 'zone': 0, 'weight': 0, 'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'}) rb.pretend_min_part_hours_passed() rb.rebalance() self.assertNotEquals(rb.validate(stats=True)[1], MAX_BALANCE) def test_get_part_devices(self): rb = ring.RingBuilder(8, 3, 1) self.assertEqual(rb.get_part_devices(0), []) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() part_devs = sorted(rb.get_part_devices(0), key=operator.itemgetter('id')) self.assertEqual(part_devs, [rb.devs[0], rb.devs[1]]) def test_get_part_devices_partial_replicas(self): rb = ring.RingBuilder(8, 2.5, 1) rb.add_dev({'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'}) rb.add_dev({'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'}) rb.rebalance() # note: partition 255 will only have 2 replicas part_devs = sorted(rb.get_part_devices(255), key=operator.itemgetter('id')) self.assertEqual(part_devs, [rb.devs[0], rb.devs[1]]) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/ring/test_ring.py0000664000175400017540000010130512323703614022625 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import array import cPickle as pickle import os import sys import unittest from contextlib import closing from gzip import GzipFile from tempfile import mkdtemp from shutil import rmtree from time import sleep, time from swift.common import ring, utils class TestRingData(unittest.TestCase): def setUp(self): self.testdir = os.path.join(os.path.dirname(__file__), 'ring_data') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) def tearDown(self): rmtree(self.testdir, ignore_errors=1) def assert_ring_data_equal(self, rd_expected, rd_got): self.assertEquals(rd_expected._replica2part2dev_id, rd_got._replica2part2dev_id) self.assertEquals(rd_expected.devs, rd_got.devs) self.assertEquals(rd_expected._part_shift, rd_got._part_shift) def test_attrs(self): r2p2d = [[0, 1, 0, 1], [0, 1, 0, 1]] d = [{'id': 0, 'zone': 0, 'region': 0, 'ip': '10.1.1.0', 'port': 7000}, {'id': 1, 'zone': 1, 'region': 1, 'ip': '10.1.1.1', 'port': 7000}] s = 30 rd = ring.RingData(r2p2d, d, s) self.assertEquals(rd._replica2part2dev_id, r2p2d) self.assertEquals(rd.devs, d) self.assertEquals(rd._part_shift, s) def test_can_load_pickled_ring_data(self): rd = ring.RingData( [[0, 1, 0, 1], [0, 1, 0, 1]], [{'id': 0, 'zone': 0, 'ip': '10.1.1.0', 'port': 7000}, {'id': 1, 'zone': 1, 'ip': '10.1.1.1', 'port': 7000}], 30) ring_fname = os.path.join(self.testdir, 'foo.ring.gz') for p in xrange(pickle.HIGHEST_PROTOCOL): with closing(GzipFile(ring_fname, 'wb')) as f: pickle.dump(rd, f, protocol=p) ring_data = ring.RingData.load(ring_fname) self.assert_ring_data_equal(rd, ring_data) def test_roundtrip_serialization(self): ring_fname = os.path.join(self.testdir, 'foo.ring.gz') rd = ring.RingData( [array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1])], [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], 30) rd.save(ring_fname) rd2 = ring.RingData.load(ring_fname) self.assert_ring_data_equal(rd, rd2) def test_deterministic_serialization(self): """ Two identical rings should produce identical .gz files on disk. Only true on Python 2.7 or greater. """ if sys.version_info[0] == 2 and sys.version_info[1] < 7: return os.mkdir(os.path.join(self.testdir, '1')) os.mkdir(os.path.join(self.testdir, '2')) # These have to have the same filename (not full path, # obviously) since the filename gets encoded in the gzip data. ring_fname1 = os.path.join(self.testdir, '1', 'the.ring.gz') ring_fname2 = os.path.join(self.testdir, '2', 'the.ring.gz') rd = ring.RingData( [array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1])], [{'id': 0, 'zone': 0}, {'id': 1, 'zone': 1}], 30) rd.save(ring_fname1) rd.save(ring_fname2) with open(ring_fname1) as ring1: with open(ring_fname2) as ring2: self.assertEqual(ring1.read(), ring2.read()) class TestRing(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' self.testdir = mkdtemp() self.testgz = os.path.join(self.testdir, 'whatever.ring.gz') self.intended_replica2part2dev_id = [ array.array('H', [0, 1, 0, 1]), array.array('H', [0, 1, 0, 1]), array.array('H', [3, 4, 3, 4])] self.intended_devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.0.1', 'replication_port': 6066}, {'id': 1, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.0.2', 'replication_port': 6066}, None, {'id': 3, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000, 'replication_ip': '10.2.0.1', 'replication_port': 6066}, {'id': 4, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000, 'replication_ip': '10.2.0.1', 'replication_port': 6066}] self.intended_part_shift = 30 self.intended_reload_time = 15 ring.RingData( self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) self.ring = ring.Ring( self.testdir, reload_time=self.intended_reload_time, ring_name='whatever') def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_creation(self): self.assertEquals(self.ring._replica2part2dev_id, self.intended_replica2part2dev_id) self.assertEquals(self.ring._part_shift, self.intended_part_shift) self.assertEquals(self.ring.devs, self.intended_devs) self.assertEquals(self.ring.reload_time, self.intended_reload_time) self.assertEquals(self.ring.serialized_path, self.testgz) # test invalid endcap _orig_hash_path_suffix = utils.HASH_PATH_SUFFIX _orig_hash_path_prefix = utils.HASH_PATH_PREFIX _orig_swift_conf_file = utils.SWIFT_CONF_FILE try: utils.HASH_PATH_SUFFIX = '' utils.HASH_PATH_PREFIX = '' utils.SWIFT_CONF_FILE = '' self.assertRaises(SystemExit, ring.Ring, self.testdir, 'whatever') finally: utils.HASH_PATH_SUFFIX = _orig_hash_path_suffix utils.HASH_PATH_PREFIX = _orig_hash_path_prefix utils.SWIFT_CONF_FILE = _orig_swift_conf_file def test_has_changed(self): self.assertEquals(self.ring.has_changed(), False) os.utime(self.testgz, (time() + 60, time() + 60)) self.assertEquals(self.ring.has_changed(), True) def test_reload(self): os.utime(self.testgz, (time() - 300, time() - 300)) self.ring = ring.Ring(self.testdir, reload_time=0.001, ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 5) self.intended_devs.append( {'id': 3, 'region': 0, 'zone': 3, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 9876}) ring.RingData( self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) self.ring.get_nodes('a') self.assertEquals(len(self.ring.devs), 6) self.assertNotEquals(self.ring._mtime, orig_mtime) os.utime(self.testgz, (time() - 300, time() - 300)) self.ring = ring.Ring(self.testdir, reload_time=0.001, ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 6) self.intended_devs.append( {'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0, 'ip': '10.5.5.5', 'port': 9876}) ring.RingData( self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) self.ring.get_part_nodes(0) self.assertEquals(len(self.ring.devs), 7) self.assertNotEquals(self.ring._mtime, orig_mtime) os.utime(self.testgz, (time() - 300, time() - 300)) self.ring = ring.Ring(self.testdir, reload_time=0.001, ring_name='whatever') orig_mtime = self.ring._mtime part, nodes = self.ring.get_nodes('a') self.assertEquals(len(self.ring.devs), 7) self.intended_devs.append( {'id': 6, 'region': 0, 'zone': 5, 'weight': 1.0, 'ip': '10.6.6.6', 'port': 6000}) ring.RingData( self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) self.ring.get_more_nodes(part).next() self.assertEquals(len(self.ring.devs), 8) self.assertNotEquals(self.ring._mtime, orig_mtime) os.utime(self.testgz, (time() - 300, time() - 300)) self.ring = ring.Ring(self.testdir, reload_time=0.001, ring_name='whatever') orig_mtime = self.ring._mtime self.assertEquals(len(self.ring.devs), 8) self.intended_devs.append( {'id': 5, 'region': 0, 'zone': 4, 'weight': 1.0, 'ip': '10.5.5.5', 'port': 6000}) ring.RingData( self.intended_replica2part2dev_id, self.intended_devs, self.intended_part_shift).save(self.testgz) sleep(0.1) self.assertEquals(len(self.ring.devs), 9) self.assertNotEquals(self.ring._mtime, orig_mtime) def test_reload_without_replication(self): replication_less_devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, {'id': 1, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, None, {'id': 3, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000}, {'id': 4, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000}] intended_devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.1.1', 'replication_port': 6000}, {'id': 1, 'region': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.1.1', 'replication_port': 6000}, None, {'id': 3, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000, 'replication_ip': '10.1.2.1', 'replication_port': 6000}, {'id': 4, 'region': 0, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000, 'replication_ip': '10.1.2.2', 'replication_port': 6000}] testgz = os.path.join(self.testdir, 'without_replication.ring.gz') ring.RingData( self.intended_replica2part2dev_id, replication_less_devs, self.intended_part_shift).save(testgz) self.ring = ring.Ring( self.testdir, reload_time=self.intended_reload_time, ring_name='without_replication') self.assertEquals(self.ring.devs, intended_devs) def test_reload_old_style_pickled_ring(self): devs = [{'id': 0, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, {'id': 1, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000}, None, {'id': 3, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000}, {'id': 4, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000}] intended_devs = [{'id': 0, 'region': 1, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.1.1', 'replication_port': 6000}, {'id': 1, 'region': 1, 'zone': 0, 'weight': 1.0, 'ip': '10.1.1.1', 'port': 6000, 'replication_ip': '10.1.1.1', 'replication_port': 6000}, None, {'id': 3, 'region': 1, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.1', 'port': 6000, 'replication_ip': '10.1.2.1', 'replication_port': 6000}, {'id': 4, 'region': 1, 'zone': 2, 'weight': 1.0, 'ip': '10.1.2.2', 'port': 6000, 'replication_ip': '10.1.2.2', 'replication_port': 6000}] # simulate an old-style pickled ring testgz = os.path.join(self.testdir, 'without_replication_or_region.ring.gz') ring_data = ring.RingData(self.intended_replica2part2dev_id, devs, self.intended_part_shift) # an old-style pickled ring won't have region data for dev in ring_data.devs: if dev: del dev["region"] gz_file = GzipFile(testgz, 'wb') pickle.dump(ring_data, gz_file, protocol=2) gz_file.close() self.ring = ring.Ring( self.testdir, reload_time=self.intended_reload_time, ring_name='without_replication_or_region') self.assertEquals(self.ring.devs, intended_devs) def test_get_part(self): part1 = self.ring.get_part('a') nodes1 = self.ring.get_part_nodes(part1) part2, nodes2 = self.ring.get_nodes('a') self.assertEquals(part1, part2) self.assertEquals(nodes1, nodes2) def test_get_part_nodes(self): part, nodes = self.ring.get_nodes('a') self.assertEquals(nodes, self.ring.get_part_nodes(part)) def test_get_nodes(self): # Yes, these tests are deliberately very fragile. We want to make sure # that if someones changes the results the ring produces, they know it. self.assertRaises(TypeError, self.ring.get_nodes) part, nodes = self.ring.get_nodes('a') self.assertEquals(part, 0) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a1') self.assertEquals(part, 0) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a4') self.assertEquals(part, 1) self.assertEquals(nodes, [self.intended_devs[1], self.intended_devs[4]]) part, nodes = self.ring.get_nodes('aa') self.assertEquals(part, 1) self.assertEquals(nodes, [self.intended_devs[1], self.intended_devs[4]]) part, nodes = self.ring.get_nodes('a', 'c1') self.assertEquals(part, 0) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a', 'c0') self.assertEquals(part, 3) self.assertEquals(nodes, [self.intended_devs[1], self.intended_devs[4]]) part, nodes = self.ring.get_nodes('a', 'c3') self.assertEquals(part, 2) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a', 'c2') self.assertEquals(part, 2) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a', 'c', 'o1') self.assertEquals(part, 1) self.assertEquals(nodes, [self.intended_devs[1], self.intended_devs[4]]) part, nodes = self.ring.get_nodes('a', 'c', 'o5') self.assertEquals(part, 0) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a', 'c', 'o0') self.assertEquals(part, 0) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) part, nodes = self.ring.get_nodes('a', 'c', 'o2') self.assertEquals(part, 2) self.assertEquals(nodes, [self.intended_devs[0], self.intended_devs[3]]) def add_dev_to_ring(self, new_dev): self.ring.devs.append(new_dev) self.ring._rebuild_tier_data() def test_get_more_nodes(self): # Yes, these tests are deliberately very fragile. We want to make sure # that if someone changes the results the ring produces, they know it. exp_part = 6 exp_devs = [48, 93, 96] exp_zones = set([5, 8, 9]) exp_handoffs = [11, 47, 25, 76, 69, 23, 99, 59, 106, 64, 43, 34, 88, 3, 30, 83, 16, 27, 103, 39, 60, 0, 8, 72, 56, 19, 91, 13, 84, 38, 66, 52, 78, 107, 50, 57, 31, 32, 77, 24, 42, 100, 71, 26, 9, 20, 35, 5, 14, 94, 28, 41, 18, 102, 101, 61, 95, 21, 81, 1, 105, 58, 74, 90, 86, 46, 4, 68, 40, 80, 54, 75, 45, 79, 44, 49, 62, 29, 7, 15, 70, 87, 65, 12, 82, 17, 104, 97, 55, 22, 6, 89, 2, 67, 37, 63, 53, 92, 33, 85, 73, 51, 98, 36, 10] exp_first_handoffs = [1, 37, 48, 68, 84, 75, 11, 101, 14, 73, 100, 75, 29, 19, 18, 101, 15, 99, 95, 24, 46, 82, 73, 62, 24, 89, 9, 22, 107, 74, 54, 63, 40, 106, 99, 83, 64, 73, 73, 106, 106, 80, 6, 25, 20, 33, 6, 79, 59, 42, 62, 24, 14, 107, 28, 0, 85, 5, 4, 12, 58, 11, 92, 18, 36, 56, 86, 1, 21, 33, 80, 97, 4, 81, 79, 76, 89, 50, 75, 27, 7, 96, 47, 55, 81, 104, 12, 5, 18, 106, 27, 93, 39, 92, 42, 30, 20, 88, 58, 105, 65, 29, 17, 52, 11, 106, 7, 24, 21, 91, 62, 52, 50, 31, 77, 102, 19, 11, 8, 58, 53, 20, 26, 8, 18, 82, 48, 68, 82, 89, 101, 50, 3, 52, 46, 11, 2, 30, 79, 66, 4, 61, 3, 56, 45, 102, 73, 84, 36, 19, 34, 84, 49, 40, 103, 66, 31, 33, 93, 33, 4, 52, 26, 58, 30, 47, 100, 57, 40, 79, 33, 107, 24, 20, 44, 4, 7, 59, 83, 101, 1, 56, 20, 61, 33, 16, 5, 74, 98, 4, 80, 15, 104, 52, 73, 18, 67, 75, 98, 73, 79, 68, 75, 27, 91, 36, 100, 52, 95, 37, 46, 70, 14, 47, 3, 70, 23, 40, 105, 62, 86, 48, 22, 54, 4, 72, 81, 13, 0, 18, 98, 101, 36, 29, 24, 39, 79, 97, 105, 28, 107, 47, 52, 101, 20, 22, 29, 65, 27, 7, 33, 64, 101, 60, 19, 55] rb = ring.RingBuilder(8, 3, 1) next_dev_id = 0 for zone in xrange(1, 10): for server in xrange(1, 5): for device in xrange(1, 4): rb.add_dev({'id': next_dev_id, 'ip': '1.2.%d.%d' % (zone, server), 'port': 1234, 'zone': zone, 'region': 0, 'weight': 1.0}) next_dev_id += 1 rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') part, devs = r.get_nodes('a', 'c', 'o') primary_zones = set([d['zone'] for d in devs]) self.assertEquals(part, exp_part) self.assertEquals([d['id'] for d in devs], exp_devs) self.assertEquals(primary_zones, exp_zones) devs = list(r.get_more_nodes(part)) self.assertEquals([d['id'] for d in devs], exp_handoffs) # The first 6 replicas plus the 3 primary nodes should cover all 9 # zones in this test seen_zones = set(primary_zones) seen_zones.update([d['zone'] for d in devs[:6]]) self.assertEquals(seen_zones, set(range(1, 10))) # The first handoff nodes for each partition in the ring devs = [] for part in xrange(r.partition_count): devs.append(r.get_more_nodes(part).next()['id']) self.assertEquals(devs, exp_first_handoffs) # Add a new device we can handoff to. zone = 5 server = 0 rb.add_dev({'id': next_dev_id, 'ip': '1.2.%d.%d' % (zone, server), 'port': 1234, 'zone': zone, 'region': 0, 'weight': 1.0}) next_dev_id += 1 rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') # We would change expectations here, but in this test no handoffs # changed at all. part, devs = r.get_nodes('a', 'c', 'o') primary_zones = set([d['zone'] for d in devs]) self.assertEquals(part, exp_part) self.assertEquals([d['id'] for d in devs], exp_devs) self.assertEquals(primary_zones, exp_zones) devs = list(r.get_more_nodes(part)) dev_ids = [d['id'] for d in devs] self.assertEquals(len(dev_ids), len(exp_handoffs)) for index, dev in enumerate(dev_ids): self.assertEquals( dev, exp_handoffs[index], 'handoff differs at position %d\n%s\n%s' % ( index, dev_ids[index:], exp_handoffs[index:])) # The handoffs still cover all the non-primary zones first seen_zones = set(primary_zones) seen_zones.update([d['zone'] for d in devs[:6]]) self.assertEquals(seen_zones, set(range(1, 10))) devs = [] for part in xrange(r.partition_count): devs.append(r.get_more_nodes(part).next()['id']) for part in xrange(r.partition_count): self.assertEquals( devs[part], exp_first_handoffs[part], 'handoff for partitition %d is now device id %d' % ( part, devs[part])) # Remove a device. rb.remove_dev(0) rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') # Change expectations # The long string of handoff nodes for the partition were the same for # the first 20, which is pretty good. exp_handoffs[20:] = [60, 108, 8, 72, 56, 19, 91, 13, 84, 38, 66, 52, 1, 78, 107, 50, 57, 31, 32, 77, 24, 42, 100, 71, 26, 9, 20, 35, 5, 14, 94, 28, 41, 18, 102, 101, 61, 95, 21, 81, 105, 58, 74, 90, 86, 46, 4, 68, 40, 80, 54, 75, 45, 79, 44, 49, 62, 29, 7, 15, 70, 87, 65, 12, 82, 17, 104, 97, 55, 22, 6, 89, 2, 67, 37, 63, 53, 92, 33, 85, 73, 51, 98, 36, 10] # Just a few of the first handoffs changed exp_first_handoffs[3] = 68 exp_first_handoffs[55] = 104 exp_first_handoffs[116] = 6 exp_first_handoffs[181] = 15 exp_first_handoffs[228] = 38 # Test part, devs = r.get_nodes('a', 'c', 'o') primary_zones = set([d['zone'] for d in devs]) self.assertEquals(part, exp_part) self.assertEquals([d['id'] for d in devs], exp_devs) self.assertEquals(primary_zones, exp_zones) devs = list(r.get_more_nodes(part)) dev_ids = [d['id'] for d in devs] self.assertEquals(len(dev_ids), len(exp_handoffs)) for index, dev in enumerate(dev_ids): self.assertEquals( dev, exp_handoffs[index], 'handoff differs at position %d\n%s\n%s' % ( index, dev_ids[index:], exp_handoffs[index:])) seen_zones = set(primary_zones) seen_zones.update([d['zone'] for d in devs[:6]]) self.assertEquals(seen_zones, set(range(1, 10))) devs = [] for part in xrange(r.partition_count): devs.append(r.get_more_nodes(part).next()['id']) for part in xrange(r.partition_count): self.assertEquals( devs[part], exp_first_handoffs[part], 'handoff for partitition %d is now device id %d' % ( part, devs[part])) # Add a partial replica rb.set_replicas(3.5) rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') # Change expectations # We have another replica now exp_devs.append(47) exp_zones.add(4) # Caused some major changes in the sequence of handoffs for our test # partition, but at least the first stayed the same. exp_handoffs[1:] = [81, 25, 69, 23, 99, 59, 76, 3, 106, 64, 43, 13, 34, 88, 30, 16, 27, 103, 39, 74, 60, 108, 8, 56, 19, 91, 52, 84, 38, 66, 1, 78, 45, 107, 50, 57, 83, 31, 46, 32, 77, 24, 42, 63, 100, 72, 71, 7, 26, 9, 20, 35, 5, 87, 14, 94, 62, 28, 41, 90, 18, 82, 102, 22, 101, 61, 85, 95, 21, 98, 67, 105, 58, 86, 4, 79, 68, 40, 80, 54, 75, 44, 49, 6, 29, 15, 70, 65, 12, 17, 104, 97, 55, 89, 2, 37, 53, 92, 33, 73, 51, 36, 10] # Lots of first handoffs changed, but 30 of 256 is still just 11.72%. exp_first_handoffs[1] = 6 exp_first_handoffs[4] = 104 exp_first_handoffs[11] = 106 exp_first_handoffs[17] = 13 exp_first_handoffs[21] = 77 exp_first_handoffs[22] = 95 exp_first_handoffs[27] = 46 exp_first_handoffs[29] = 65 exp_first_handoffs[30] = 3 exp_first_handoffs[31] = 20 exp_first_handoffs[51] = 50 exp_first_handoffs[53] = 8 exp_first_handoffs[54] = 2 exp_first_handoffs[72] = 107 exp_first_handoffs[79] = 72 exp_first_handoffs[85] = 71 exp_first_handoffs[88] = 66 exp_first_handoffs[92] = 29 exp_first_handoffs[93] = 46 exp_first_handoffs[96] = 38 exp_first_handoffs[101] = 57 exp_first_handoffs[103] = 87 exp_first_handoffs[104] = 28 exp_first_handoffs[107] = 1 exp_first_handoffs[109] = 69 exp_first_handoffs[110] = 50 exp_first_handoffs[111] = 76 exp_first_handoffs[115] = 47 exp_first_handoffs[117] = 48 exp_first_handoffs[119] = 7 # Test part, devs = r.get_nodes('a', 'c', 'o') primary_zones = set([d['zone'] for d in devs]) self.assertEquals(part, exp_part) self.assertEquals([d['id'] for d in devs], exp_devs) self.assertEquals(primary_zones, exp_zones) devs = list(r.get_more_nodes(part)) dev_ids = [d['id'] for d in devs] self.assertEquals(len(dev_ids), len(exp_handoffs)) for index, dev in enumerate(dev_ids): self.assertEquals( dev, exp_handoffs[index], 'handoff differs at position %d\n%s\n%s' % ( index, dev_ids[index:], exp_handoffs[index:])) seen_zones = set(primary_zones) seen_zones.update([d['zone'] for d in devs[:6]]) self.assertEquals(seen_zones, set(range(1, 10))) devs = [] for part in xrange(r.partition_count): devs.append(r.get_more_nodes(part).next()['id']) for part in xrange(r.partition_count): self.assertEquals( devs[part], exp_first_handoffs[part], 'handoff for partitition %d is now device id %d' % ( part, devs[part])) # One last test of a partial replica partition exp_part2 = 136 exp_devs2 = [52, 76, 97] exp_zones2 = set([9, 5, 7]) exp_handoffs2 = [2, 67, 37, 92, 33, 23, 107, 63, 44, 103, 108, 85, 73, 10, 89, 80, 4, 17, 49, 32, 12, 41, 58, 20, 25, 61, 94, 47, 69, 56, 101, 28, 83, 8, 96, 53, 51, 42, 98, 35, 36, 84, 43, 104, 31, 65, 1, 40, 9, 74, 95, 45, 5, 71, 86, 78, 30, 93, 48, 91, 15, 88, 39, 18, 57, 72, 70, 27, 54, 16, 24, 21, 14, 11, 77, 62, 50, 6, 105, 26, 55, 29, 60, 34, 13, 87, 59, 38, 99, 75, 106, 3, 82, 66, 79, 7, 46, 64, 81, 22, 68, 19, 102, 90, 100] part2, devs2 = r.get_nodes('a', 'c', 'o2') primary_zones2 = set([d['zone'] for d in devs2]) self.assertEquals(part2, exp_part2) self.assertEquals([d['id'] for d in devs2], exp_devs2) self.assertEquals(primary_zones2, exp_zones2) devs2 = list(r.get_more_nodes(part2)) dev_ids2 = [d['id'] for d in devs2] self.assertEquals(len(dev_ids2), len(exp_handoffs2)) for index, dev in enumerate(dev_ids2): self.assertEquals( dev, exp_handoffs2[index], 'handoff differs at position %d\n%s\n%s' % ( index, dev_ids2[index:], exp_handoffs2[index:])) seen_zones = set(primary_zones2) seen_zones.update([d['zone'] for d in devs2[:6]]) self.assertEquals(seen_zones, set(range(1, 10))) # Test distribution across regions rb.set_replicas(3) for region in xrange(1, 5): rb.add_dev({'id': next_dev_id, 'ip': '1.%d.1.%d' % (region, server), 'port': 1234, 'zone': 1, 'region': region, 'weight': 1.0}) next_dev_id += 1 rb.pretend_min_part_hours_passed() rb.rebalance(seed=1) rb.pretend_min_part_hours_passed() rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') # There's 5 regions now, so the primary nodes + first 2 handoffs # should span all 5 regions part, devs = r.get_nodes('a1', 'c1', 'o1') primary_regions = set([d['region'] for d in devs]) primary_zones = set([(d['region'], d['zone']) for d in devs]) more_devs = list(r.get_more_nodes(part)) seen_regions = set(primary_regions) seen_regions.update([d['region'] for d in more_devs[:2]]) self.assertEquals(seen_regions, set(range(0, 5))) # There are 13 zones now, so the first 13 nodes should all have # distinct zones (that's r0z0, r0z1, ..., r0z8, r1z1, r2z1, r3z1, and # r4z1). seen_zones = set(primary_zones) seen_zones.update([(d['region'], d['zone']) for d in more_devs[:10]]) self.assertEquals(13, len(seen_zones)) # Here's a brittle canary-in-the-coalmine test to make sure the region # handoff computation didn't change accidentally exp_handoffs = [111, 112, 74, 54, 93, 31, 2, 43, 100, 22, 71, 92, 35, 9, 50, 41, 76, 80, 84, 88, 17, 96, 6, 102, 37, 29, 105, 5, 47, 20, 13, 108, 66, 81, 53, 65, 25, 58, 32, 94, 101, 1, 10, 44, 73, 75, 21, 97, 28, 106, 30, 16, 39, 77, 42, 72, 34, 99, 14, 61, 90, 4, 40, 3, 45, 62, 7, 15, 87, 12, 83, 89, 33, 98, 49, 107, 56, 86, 48, 57, 24, 11, 23, 26, 46, 64, 69, 38, 36, 79, 63, 104, 51, 70, 82, 67, 68, 8, 95, 91, 55, 59, 85] dev_ids = [d['id'] for d in more_devs] self.assertEquals(len(dev_ids), len(exp_handoffs)) for index, dev_id in enumerate(dev_ids): self.assertEquals( dev_id, exp_handoffs[index], 'handoff differs at position %d\n%s\n%s' % ( index, dev_ids[index:], exp_handoffs[index:])) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/ring/test_utils.py0000664000175400017540000001603012323703611023023 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.common.ring.utils import (build_tier_tree, tiers_for_dev, parse_search_value, parse_args, build_dev_from_opts, parse_builder_ring_filename_args) class TestUtils(unittest.TestCase): def setUp(self): self.test_dev = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 0} def get_test_devs(): dev0 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 0} dev1 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 1} dev2 = {'region': 1, 'zone': 1, 'ip': '192.168.1.1', 'port': '6000', 'id': 2} dev3 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 3} dev4 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 4} dev5 = {'region': 1, 'zone': 1, 'ip': '192.168.1.2', 'port': '6000', 'id': 5} dev6 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 6} dev7 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 7} dev8 = {'region': 1, 'zone': 2, 'ip': '192.168.2.1', 'port': '6000', 'id': 8} dev9 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 9} dev10 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 10} dev11 = {'region': 1, 'zone': 2, 'ip': '192.168.2.2', 'port': '6000', 'id': 11} return [dev0, dev1, dev2, dev3, dev4, dev5, dev6, dev7, dev8, dev9, dev10, dev11] self.test_devs = get_test_devs() def test_tiers_for_dev(self): self.assertEqual( tiers_for_dev(self.test_dev), ((1,), (1, 1), (1, 1, '192.168.1.1:6000'), (1, 1, '192.168.1.1:6000', 0))) def test_build_tier_tree(self): ret = build_tier_tree(self.test_devs) self.assertEqual(len(ret), 8) self.assertEqual(ret[()], set([(1,)])) self.assertEqual(ret[(1,)], set([(1, 1), (1, 2)])) self.assertEqual(ret[(1, 1)], set([(1, 1, '192.168.1.2:6000'), (1, 1, '192.168.1.1:6000')])) self.assertEqual(ret[(1, 2)], set([(1, 2, '192.168.2.2:6000'), (1, 2, '192.168.2.1:6000')])) self.assertEqual(ret[(1, 1, '192.168.1.1:6000')], set([(1, 1, '192.168.1.1:6000', 0), (1, 1, '192.168.1.1:6000', 1), (1, 1, '192.168.1.1:6000', 2)])) self.assertEqual(ret[(1, 1, '192.168.1.2:6000')], set([(1, 1, '192.168.1.2:6000', 3), (1, 1, '192.168.1.2:6000', 4), (1, 1, '192.168.1.2:6000', 5)])) self.assertEqual(ret[(1, 2, '192.168.2.1:6000')], set([(1, 2, '192.168.2.1:6000', 6), (1, 2, '192.168.2.1:6000', 7), (1, 2, '192.168.2.1:6000', 8)])) self.assertEqual(ret[(1, 2, '192.168.2.2:6000')], set([(1, 2, '192.168.2.2:6000', 9), (1, 2, '192.168.2.2:6000', 10), (1, 2, '192.168.2.2:6000', 11)])) def test_parse_search_value(self): res = parse_search_value('r0') self.assertEqual(res, {'region': 0}) res = parse_search_value('r1') self.assertEqual(res, {'region': 1}) res = parse_search_value('r1z2') self.assertEqual(res, {'region': 1, 'zone': 2}) res = parse_search_value('d1') self.assertEqual(res, {'id': 1}) res = parse_search_value('z1') self.assertEqual(res, {'zone': 1}) res = parse_search_value('-127.0.0.1') self.assertEqual(res, {'ip': '127.0.0.1'}) res = parse_search_value('-[127.0.0.1]:10001') self.assertEqual(res, {'ip': '127.0.0.1', 'port': 10001}) res = parse_search_value(':10001') self.assertEqual(res, {'port': 10001}) res = parse_search_value('R127.0.0.10') self.assertEqual(res, {'replication_ip': '127.0.0.10'}) res = parse_search_value('R[127.0.0.10]:20000') self.assertEqual(res, {'replication_ip': '127.0.0.10', 'replication_port': 20000}) res = parse_search_value('R:20000') self.assertEqual(res, {'replication_port': 20000}) res = parse_search_value('/sdb1') self.assertEqual(res, {'device': 'sdb1'}) res = parse_search_value('_meta1') self.assertEqual(res, {'meta': 'meta1'}) self.assertRaises(ValueError, parse_search_value, 'OMGPONIES') def test_replication_defaults(self): args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split() opts, _ = parse_args(args) device = build_dev_from_opts(opts) expected = { 'device': 'd1', 'ip': '127.0.0.1', 'meta': '', 'port': 6010, 'region': 1, 'replication_ip': '127.0.0.1', 'replication_port': 6010, 'weight': 100.0, 'zone': 1, } self.assertEquals(device, expected) def test_parse_builder_ring_filename_args(self): args = 'swift-ring-builder object.builder write_ring' self.assertEquals(( 'object.builder', 'object.ring.gz' ), parse_builder_ring_filename_args(args.split())) args = 'swift-ring-builder container.ring.gz write_builder' self.assertEquals(( 'container.builder', 'container.ring.gz' ), parse_builder_ring_filename_args(args.split())) # builer name arg should always fall through args = 'swift-ring-builder test create' self.assertEquals(( 'test', 'test.ring.gz' ), parse_builder_ring_filename_args(args.split())) args = 'swift-ring-builder my.file.name create' self.assertEquals(( 'my.file.name', 'my.file.name.ring.gz' ), parse_builder_ring_filename_args(args.split())) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/common/ring/__init__.py0000664000175400017540000000000012323703611022351 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/common/__init__.py0000664000175400017540000000000012323703611021412 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/container/0000775000175400017540000000000012323703665020016 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/container/test_server.py0000664000175400017540000025416712323703614022746 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import operator import os import mock import unittest from contextlib import contextmanager from shutil import rmtree from StringIO import StringIO from tempfile import mkdtemp from test.unit import FakeLogger from xml.dom import minidom from eventlet import spawn, Timeout, listen import simplejson from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server from swift.common.utils import (normalize_timestamp, mkdirs, public, replication, lock_parent_directory) from test.unit import fake_http_connect from swift.common.request_helpers import get_sys_meta_prefix @contextmanager def save_globals(): orig_http_connect = getattr(swift.container.server, 'http_connect', None) try: yield True finally: swift.container.server.http_connect = orig_http_connect class TestContainerController(unittest.TestCase): """Test swift.container.server.ContainerController""" def setUp(self): """Set up for testing swift.object_server.ObjectController""" self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(self.testdir) rmtree(self.testdir) mkdirs(os.path.join(self.testdir, 'sda1')) mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) self.controller = container_server.ContainerController( {'devices': self.testdir, 'mount_check': 'false'}) def tearDown(self): """Tear down for testing swift.object_server.ObjectController""" rmtree(os.path.dirname(self.testdir), ignore_errors=1) def test_acl_container(self): # Ensure no acl by default req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0'}) resp = req.get_response(self.controller) self.assert_(resp.status.startswith('201')) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) response = req.get_response(self.controller) self.assert_(response.status.startswith('204')) self.assert_('x-container-read' not in response.headers) self.assert_('x-container-write' not in response.headers) # Ensure POSTing acls works req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': '1', 'X-Container-Read': '.r:*', 'X-Container-Write': 'account:user'}) resp = req.get_response(self.controller) self.assert_(resp.status.startswith('204')) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) response = req.get_response(self.controller) self.assert_(response.status.startswith('204')) self.assertEquals(response.headers.get('x-container-read'), '.r:*') self.assertEquals(response.headers.get('x-container-write'), 'account:user') # Ensure we can clear acls on POST req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': '3', 'X-Container-Read': '', 'X-Container-Write': ''}) resp = req.get_response(self.controller) self.assert_(resp.status.startswith('204')) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) response = req.get_response(self.controller) self.assert_(response.status.startswith('204')) self.assert_('x-container-read' not in response.headers) self.assert_('x-container-write' not in response.headers) # Ensure PUTing acls works req = Request.blank( '/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '4', 'X-Container-Read': '.r:*', 'X-Container-Write': 'account:user'}) resp = req.get_response(self.controller) self.assert_(resp.status.startswith('201')) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'}) response = req.get_response(self.controller) self.assert_(response.status.startswith('204')) self.assertEquals(response.headers.get('x-container-read'), '.r:*') self.assertEquals(response.headers.get('x-container-write'), 'account:user') def test_HEAD(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '0'}) response = req.get_response(self.controller) self.assert_(response.status.startswith('204')) self.assertEquals(int(response.headers['x-container-bytes-used']), 0) self.assertEquals(int(response.headers['x-container-object-count']), 0) req2 = Request.blank( '/sda1/p/a/c/o', environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_SIZE': 42, 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'}) req2.get_response(self.controller) response = req.get_response(self.controller) self.assertEquals(int(response.headers['x-container-bytes-used']), 42) self.assertEquals(int(response.headers['x-container-object-count']), 1) def test_HEAD_not_found(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_HEAD_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_HEAD_insufficient_storage(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'HEAD', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 507) def test_HEAD_invalid_content_type(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}, headers={'Accept': 'application/plain'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 406) def test_HEAD_invalid_format(self): format = '%D1%BD%8A9' # invalid UTF-8; should be %E1%BD%8A9 (E -> D) req = Request.blank( '/sda1/p/a/c?format=' + format, environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_PUT(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) def test_PUT_simulated_create_race(self): state = ['initial'] from swift.container.backend import ContainerBroker as OrigCoBr class InterceptedCoBr(OrigCoBr): def __init__(self, *args, **kwargs): super(InterceptedCoBr, self).__init__(*args, **kwargs) if state[0] == 'initial': # Do nothing initially pass elif state[0] == 'race': # Save the original db_file attribute value self._saved_db_file = self.db_file self.db_file += '.doesnotexist' def initialize(self, *args, **kwargs): if state[0] == 'initial': # Do nothing initially pass elif state[0] == 'race': # Restore the original db_file attribute to get the race # behavior self.db_file = self._saved_db_file return super(InterceptedCoBr, self).initialize(*args, **kwargs) with mock.patch("swift.container.server.ContainerBroker", InterceptedCoBr): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 201) state[0] = "race" req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 202) def test_PUT_obj_not_found(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1', 'X-Size': '0', 'X-Content-Type': 'text/plain', 'X-ETag': 'e'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_PUT_GET_metadata(self): # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Container-Meta-Test2': 'Value2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value') self.assertEquals(resp.headers.get('x-container-meta-test2'), 'Value2') # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(3), 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(2), 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(4), 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) def test_PUT_GET_sys_metadata(self): prefix = get_sys_meta_prefix('container') key = '%sTest' % prefix key2 = '%sTest2' % prefix # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1), key: 'Value'}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c') resp = self.controller.GET(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'Value') # Set another metadata header, ensuring old one doesn't disappear req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), key2: 'Value2'}) resp = self.controller.POST(req) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c') resp = self.controller.GET(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'Value') self.assertEquals(resp.headers.get(key2.lower()), 'Value2') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(3), key: 'New Value'}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c') resp = self.controller.GET(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(2), key: 'Old Value'}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c') resp = self.controller.GET(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(4), key: ''}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c') resp = self.controller.GET(req) self.assertEquals(resp.status_int, 204) self.assert_(key.lower() not in resp.headers) def test_PUT_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_PUT_timestamp_not_float(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 'not-float'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_PUT_insufficient_storage(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 507) def test_POST_HEAD_metadata(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), 'X-Container-Meta-Test': 'Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'Value') # Update metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(3), 'X-Container-Meta-Test': 'New Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'New Value') # Send old update to metadata header req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(2), 'X-Container-Meta-Test': 'Old Value'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get('x-container-meta-test'), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(4), 'X-Container-Meta-Test': ''}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) self.assert_('x-container-meta-test' not in resp.headers) def test_POST_HEAD_sys_metadata(self): prefix = get_sys_meta_prefix('container') key = '%sTest' % prefix req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(1)}) resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 201) # Set metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(1), key: 'Value'}) resp = self.controller.POST(req) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = self.controller.HEAD(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'Value') # Update metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(3), key: 'New Value'}) resp = self.controller.POST(req) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = self.controller.HEAD(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'New Value') # Send old update to metadata header req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(2), key: 'Old Value'}) resp = self.controller.POST(req) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = self.controller.HEAD(req) self.assertEquals(resp.status_int, 204) self.assertEquals(resp.headers.get(key.lower()), 'New Value') # Remove metadata header (by setting it to empty) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(4), key: ''}) resp = self.controller.POST(req) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = self.controller.HEAD(req) self.assertEquals(resp.status_int, 204) self.assert_(key.lower() not in resp.headers) def test_POST_invalid_partition(self): req = Request.blank('/sda1/./a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_POST_timestamp_not_float(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': 'not-float'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_POST_insufficient_storage(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 507) def test_POST_invalid_container_sync_to(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'POST', 'HTTP_X_TIMESTAMP': '1'}, headers={'x-container-sync-to': '192.168.0.1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_POST_after_DELETE_not_found(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) req = Request.blank('/sda1/p/a/c/', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': '3'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_DELETE_obj_not_found(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_DELETE_container_not_found(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_PUT_utf8(self): snowman = u'\u2603' container_name = snowman.encode('utf-8') req = Request.blank( '/sda1/p/a/%s' % container_name, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) def test_account_update_mismatched_host_device(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}, headers={'X-Timestamp': '0000000001.00000', 'X-Account-Host': '127.0.0.1:0', 'X-Account-Partition': '123', 'X-Account-Device': 'sda1,sda2'}) broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c') resp = self.controller.account_update(req, 'a', 'c', broker) self.assertEquals(resp.status_int, 400) def test_account_update_account_override_deleted(self): bindsock = listen(('127.0.0.1', 0)) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1'}, headers={'X-Timestamp': '0000000001.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1', 'X-Account-Override-Deleted': 'yes'}) with save_globals(): new_connect = fake_http_connect(200, count=123) swift.container.server.http_connect = new_connect resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) def test_PUT_account_update(self): bindsock = listen(('127.0.0.1', 0)) def accept(return_code, expected_timestamp): try: with Timeout(3): sock, addr = bindsock.accept() inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/123/a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assertEquals(headers['x-put-timestamp'], expected_timestamp) except BaseException as err: return err return None req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0000000001.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 201, '0000000001.00000') try: with Timeout(3): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) finally: err = event.wait() if err: raise Exception(err) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0000000003.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 404, '0000000003.00000') try: with Timeout(3): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) finally: err = event.wait() if err: raise Exception(err) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '0000000005.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 503, '0000000005.00000') got_exc = False try: with Timeout(3): resp = req.get_response(self.controller) except BaseException as err: got_exc = True finally: err = event.wait() if err: raise Exception(err) self.assert_(not got_exc) def test_PUT_reset_container_sync(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) db.set_x_container_sync_points(123, 456) info = db.get_info() self.assertEquals(info['x_container_sync_point1'], 123) self.assertEquals(info['x_container_sync_point2'], 456) # Set to same value req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], 123) self.assertEquals(info['x_container_sync_point2'], 456) # Set to new value req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 202) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) def test_POST_reset_container_sync(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) db.set_x_container_sync_points(123, 456) info = db.get_info() self.assertEquals(info['x_container_sync_point1'], 123) self.assertEquals(info['x_container_sync_point2'], 456) # Set to same value req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], 123) self.assertEquals(info['x_container_sync_point2'], 456) # Set to new value req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'x-timestamp': '1', 'x-container-sync-to': 'http://127.0.0.1:12345/v1/a/c2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') info = db.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) def test_DELETE(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '3'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_DELETE_PUT_recreate(self): path = '/sda1/p/a/c' req = Request.blank(path, method='PUT', headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank(path, method='DELETE', headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank(path, method='GET') resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) # sanity db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(True, db.is_deleted()) info = db.get_info() self.assertEquals(info['put_timestamp'], normalize_timestamp('1')) self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) # recreate req = Request.blank(path, method='PUT', headers={'X-Timestamp': '4'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(False, db.is_deleted()) info = db.get_info() self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) def test_DELETE_PUT_recreate_replication_race(self): path = '/sda1/p/a/c' # create a deleted db req = Request.blank(path, method='PUT', headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') req = Request.blank(path, method='DELETE', headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank(path, method='GET') resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) # sanity self.assertEqual(True, db.is_deleted()) # now save a copy of this db (and remove it from the "current node") db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') db_path = db.db_file other_path = os.path.join(self.testdir, 'othernode.db') os.rename(db_path, other_path) # that should make it missing on this node req = Request.blank(path, method='GET') resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) # sanity # setup the race in os.path.exists (first time no, then yes) mock_called = [] _real_exists = os.path.exists def mock_exists(db_path): rv = _real_exists(db_path) if not mock_called: # be as careful as we might hope backend replication can be... with lock_parent_directory(db_path, timeout=1): os.rename(other_path, db_path) mock_called.append((rv, db_path)) return rv req = Request.blank(path, method='PUT', headers={'X-Timestamp': '4'}) with mock.patch.object(container_server.os.path, 'exists', mock_exists): resp = req.get_response(self.controller) # db was successfully created self.assertEqual(resp.status_int // 100, 2) db = self.controller._get_container_broker('sda1', 'p', 'a', 'c') self.assertEqual(False, db.is_deleted()) # mock proves the race self.assertEqual(mock_called[:2], [(exists, db.db_file) for exists in (False, True)]) # info was updated info = db.get_info() self.assertEquals(info['put_timestamp'], normalize_timestamp('4')) self.assertEquals(info['delete_timestamp'], normalize_timestamp('2')) def test_DELETE_not_found(self): # Even if the container wasn't previously heard of, the container # server will accept the delete and replicate it to where it belongs # later. req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_DELETE_object(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0', 'HTTP_X_SIZE': 1, 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '3'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 409) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '4'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '5'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': '6'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_DELETE_account_update(self): bindsock = listen(('127.0.0.1', 0)) def accept(return_code, expected_timestamp): try: with Timeout(3): sock, addr = bindsock.accept() inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/123/a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assertEquals(headers['x-delete-timestamp'], expected_timestamp) except BaseException as err: return err return None req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '0000000002.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 204, '0000000002.00000') try: with Timeout(3): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) finally: err = event.wait() if err: raise Exception(err) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '2'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '0000000003.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 404, '0000000003.00000') try: with Timeout(3): resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) finally: err = event.wait() if err: raise Exception(err) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '4'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': '0000000005.00000', 'X-Account-Host': '%s:%s' % bindsock.getsockname(), 'X-Account-Partition': '123', 'X-Account-Device': 'sda1'}) event = spawn(accept, 503, '0000000005.00000') got_exc = False try: with Timeout(3): resp = req.get_response(self.controller) except BaseException as err: got_exc = True finally: err = event.wait() if err: raise Exception(err) self.assert_(not got_exc) def test_DELETE_invalid_partition(self): req = Request.blank( '/sda1/./a/c', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_DELETE_timestamp_not_float(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': 'not-float'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400) def test_DELETE_insufficient_storage(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'DELETE', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 507) def test_GET_over_limit(self): req = Request.blank( '/sda1/p/a/c?limit=%d' % (container_server.CONTAINER_LISTING_LIMIT + 1), environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 412) def test_GET_json(self): # make a container req = Request.blank( '/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) # test an empty container req = Request.blank( '/sda1/p/a/jsonc?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 200) self.assertEquals(simplejson.loads(resp.body), []) # fill the container for i in range(3): req = Request.blank( '/sda1/p/a/jsonc/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # test format json_body = [{"name": "0", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}, {"name": "1", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}, {"name": "2", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}] req = Request.blank( '/sda1/p/a/jsonc?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(simplejson.loads(resp.body), json_body) self.assertEquals(resp.charset, 'utf-8') req = Request.blank( '/sda1/p/a/jsonc?format=json', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/json') for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/json;q=1.0', 'application/*'): req = Request.blank( '/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'GET'}) req.accept = accept resp = req.get_response(self.controller) self.assertEquals( simplejson.loads(resp.body), json_body, 'Invalid body for Accept: %s' % accept) self.assertEquals( resp.content_type, 'application/json', 'Invalid content_type for Accept: %s' % accept) req = Request.blank( '/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'HEAD'}) req.accept = accept resp = req.get_response(self.controller) self.assertEquals( resp.content_type, 'application/json', 'Invalid content_type for Accept: %s' % accept) def test_GET_plain(self): # make a container req = Request.blank( '/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) # test an empty container req = Request.blank( '/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) # fill the container for i in range(3): req = Request.blank( '/sda1/p/a/plainc/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) plain_body = '0\n1\n2\n' req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.body, plain_body) self.assertEquals(resp.charset, 'utf-8') req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'text/plain') for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', '*/*;q=0.9,application/xml;q=0.8', '*/*', 'text/plain,application/xml'): req = Request.blank( '/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) req.accept = accept resp = req.get_response(self.controller) self.assertEquals( resp.body, plain_body, 'Invalid body for Accept: %s' % accept) self.assertEquals( resp.content_type, 'text/plain', 'Invalid content_type for Accept: %s' % accept) req = Request.blank( '/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'}) req.accept = accept resp = req.get_response(self.controller) self.assertEquals( resp.content_type, 'text/plain', 'Invalid content_type for Accept: %s' % accept) # test conflicting formats req = Request.blank( '/sda1/p/a/plainc?format=plain', environ={'REQUEST_METHOD': 'GET'}) req.accept = 'application/json' resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.body, plain_body) # test unknown format uses default plain req = Request.blank( '/sda1/p/a/plainc?format=somethingelse', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.body, plain_body) def test_GET_json_last_modified(self): # make a container req = Request.blank( '/sda1/p/a/jsonc', environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i, d in [(0, 1.5), (1, 1.0), ]: req = Request.blank( '/sda1/p/a/jsonc/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': d, 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # test format # last_modified format must be uniform, even when there are not msecs json_body = [{"name": "0", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.500000"}, {"name": "1", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}, ] req = Request.blank( '/sda1/p/a/jsonc?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(simplejson.loads(resp.body), json_body) self.assertEquals(resp.charset, 'utf-8') def test_GET_xml(self): # make a container req = Request.blank( '/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) # fill the container for i in range(3): req = Request.blank( '/sda1/p/a/xmlc/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) xml_body = '\n' \ '' \ '0x0' \ 'text/plain' \ '1970-01-01T00:00:01.000000' \ '' \ '1x0' \ 'text/plain' \ '1970-01-01T00:00:01.000000' \ '' \ '2x0' \ 'text/plain' \ '1970-01-01T00:00:01.000000' \ '' \ '' # tests req = Request.blank( '/sda1/p/a/xmlc?format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/xml') self.assertEquals(resp.body, xml_body) self.assertEquals(resp.charset, 'utf-8') req = Request.blank( '/sda1/p/a/xmlc?format=xml', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/xml') for xml_accept in ( 'application/xml', 'application/xml;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'): req = Request.blank( '/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'GET'}) req.accept = xml_accept resp = req.get_response(self.controller) self.assertEquals( resp.body, xml_body, 'Invalid body for Accept: %s' % xml_accept) self.assertEquals( resp.content_type, 'application/xml', 'Invalid content_type for Accept: %s' % xml_accept) req = Request.blank( '/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'HEAD'}) req.accept = xml_accept resp = req.get_response(self.controller) self.assertEquals( resp.content_type, 'application/xml', 'Invalid content_type for Accept: %s' % xml_accept) req = Request.blank( '/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'GET'}) req.accept = 'text/xml' resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'text/xml') self.assertEquals(resp.body, xml_body) def test_GET_marker(self): # make a container req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) # fill the container for i in range(3): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # test limit with marker req = Request.blank('/sda1/p/a/c?limit=2&marker=1', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) result = resp.body.split() self.assertEquals(result, ['2', ]) def test_weird_content_types(self): snowman = u'\u2603' req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i, ctype in enumerate((snowman.encode('utf-8'), 'text/plain; charset="utf-8"')): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype, 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) result = [x['content_type'] for x in simplejson.loads(resp.body)] self.assertEquals(result, [u'\u2603', 'text/plain;charset="utf-8"']) def test_GET_accept_not_valid(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Put-Timestamp': '1', 'X-Delete-Timestamp': '0', 'X-Object-Count': '0', 'X-Bytes-Used': '0', 'X-Timestamp': normalize_timestamp(0)}) req.get_response(self.controller) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) req.accept = 'application/xml*' resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 406) def test_GET_limit(self): # make a container req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) # fill the container for i in range(3): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) # test limit req = Request.blank( '/sda1/p/a/c?limit=2', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) result = resp.body.split() self.assertEquals(result, ['0', '1']) def test_GET_prefix(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i in ('a1', 'b1', 'a2', 'b2', 'a3', 'b3'): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c?prefix=a', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.body.split(), ['a1', 'a2', 'a3']) def test_GET_delimiter_too_long(self): req = Request.blank('/sda1/p/a/c?delimiter=xx', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 412) def test_GET_delimiter(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c?prefix=US-&delimiter=-&format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals( simplejson.loads(resp.body), [{"subdir": "US-OK-"}, {"subdir": "US-TX-"}, {"subdir": "US-UT-"}]) def test_GET_delimiter_xml(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i in ('US-TX-A', 'US-TX-B', 'US-OK-A', 'US-OK-B', 'US-UT-A'): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c?prefix=US-&delimiter=-&format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals( resp.body, '' '\n' 'US-OK-' 'US-TX-' 'US-UT-') def test_GET_delimiter_xml_with_quotes(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) req = Request.blank( '/sda1/p/a/c/<\'sub\' "dir">/object', environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c?delimiter=/&format=xml', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) dom = minidom.parseString(resp.body) self.assert_(len(dom.getElementsByTagName('container')) == 1) container = dom.getElementsByTagName('container')[0] self.assert_(len(container.getElementsByTagName('subdir')) == 1) subdir = container.getElementsByTagName('subdir')[0] self.assertEquals(unicode(subdir.attributes['name'].value), u'<\'sub\' "dir">/') self.assert_(len(subdir.getElementsByTagName('name')) == 1) name = subdir.getElementsByTagName('name')[0] self.assertEquals(unicode(name.childNodes[0].data), u'<\'sub\' "dir">/') def test_GET_path(self): req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(self.controller) for i in ('US/TX', 'US/TX/B', 'US/OK', 'US/OK/B', 'US/UT/A'): req = Request.blank( '/sda1/p/a/c/%s' % i, environ={ 'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': 'text/plain', 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 0}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c?path=US&format=json', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals( simplejson.loads(resp.body), [{"name": "US/OK", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}, {"name": "US/TX", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}]) def test_GET_insufficient_storage(self): self.controller = container_server.ContainerController( {'devices': self.testdir}) req = Request.blank( '/sda-null/p/a/c', environ={'REQUEST_METHOD': 'GET', 'HTTP_X_TIMESTAMP': '1'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 507) def test_through_call(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '404 ') def test_through_call_invalid_path(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/bob', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '400 ') def test_through_call_invalid_path_utf8(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '\x00', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '412 ') def test_invalid_method_doesnt_exist(self): errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.controller.__call__({'REQUEST_METHOD': 'method_doesnt_exist', 'PATH_INFO': '/sda1/p/a/c'}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_invalid_method_is_not_public(self): errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.controller.__call__({'REQUEST_METHOD': '__init__', 'PATH_INFO': '/sda1/p/a/c'}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_params_format(self): req = Request.blank( '/sda1/p/a/c', headers={'X-Timestamp': normalize_timestamp(1)}, environ={'REQUEST_METHOD': 'PUT'}) req.get_response(self.controller) for format in ('xml', 'json'): req = Request.blank('/sda1/p/a/c?format=%s' % format, environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 200) def test_params_utf8(self): # Bad UTF8 sequence, all parameters should cause 400 error for param in ('delimiter', 'limit', 'marker', 'path', 'prefix', 'end_marker', 'format'): req = Request.blank('/sda1/p/a/c?%s=\xce' % param, environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 400, "%d on param %s" % (resp.status_int, param)) # Good UTF8 sequence for delimiter, too long (1 byte delimiters only) req = Request.blank('/sda1/p/a/c?delimiter=\xce\xa9', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 412, "%d on param delimiter" % (resp.status_int)) req = Request.blank('/sda1/p/a/c', headers={'X-Timestamp': normalize_timestamp(1)}, environ={'REQUEST_METHOD': 'PUT'}) req.get_response(self.controller) # Good UTF8 sequence, ignored for limit, doesn't affect other queries for param in ('limit', 'marker', 'path', 'prefix', 'end_marker', 'format'): req = Request.blank('/sda1/p/a/c?%s=\xce\xa9' % param, environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204, "%d on param %s" % (resp.status_int, param)) def test_put_auto_create(self): headers = {'x-timestamp': normalize_timestamp(1), 'x-size': '0', 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e'} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/.a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/.c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/a/c/.o', environ={'REQUEST_METHOD': 'PUT'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_delete_auto_create(self): headers = {'x-timestamp': normalize_timestamp(1)} req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/.a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/.c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/a/.c/.o', environ={'REQUEST_METHOD': 'DELETE'}, headers=dict(headers)) resp = req.get_response(self.controller) self.assertEquals(resp.status_int, 404) def test_content_type_on_HEAD(self): Request.blank('/sda1/p/a/o', headers={'X-Timestamp': normalize_timestamp(1)}, environ={'REQUEST_METHOD': 'PUT'}).get_response( self.controller) env = {'REQUEST_METHOD': 'HEAD'} req = Request.blank('/sda1/p/a/o?format=xml', environ=env) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/xml') self.assertEquals(resp.charset, 'utf-8') req = Request.blank('/sda1/p/a/o?format=json', environ=env) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(resp.charset, 'utf-8') req = Request.blank('/sda1/p/a/o', environ=env) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.charset, 'utf-8') req = Request.blank( '/sda1/p/a/o', headers={'Accept': 'application/json'}, environ=env) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(resp.charset, 'utf-8') req = Request.blank( '/sda1/p/a/o', headers={'Accept': 'application/xml'}, environ=env) resp = req.get_response(self.controller) self.assertEquals(resp.content_type, 'application/xml') self.assertEquals(resp.charset, 'utf-8') def test_updating_multiple_container_servers(self): http_connect_args = [] def fake_http_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): class SuccessfulFakeConn(object): @property def status(self): return 200 def getresponse(self): return self def read(self): return '' captured_args = {'ipaddr': ipaddr, 'port': port, 'device': device, 'partition': partition, 'method': method, 'path': path, 'ssl': ssl, 'headers': headers, 'query_string': query_string} http_connect_args.append( dict((k, v) for k, v in captured_args.iteritems() if v is not None)) req = Request.blank( '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '12345', 'X-Account-Partition': '30', 'X-Account-Host': '1.2.3.4:5, 6.7.8.9:10', 'X-Account-Device': 'sdb1, sdf1'}) orig_http_connect = container_server.http_connect try: container_server.http_connect = fake_http_connect req.get_response(self.controller) finally: container_server.http_connect = orig_http_connect http_connect_args.sort(key=operator.itemgetter('ipaddr')) self.assertEquals(len(http_connect_args), 2) self.assertEquals( http_connect_args[0], {'ipaddr': '1.2.3.4', 'port': '5', 'path': '/a/c', 'device': 'sdb1', 'partition': '30', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, 'x-put-timestamp': '0000012345.00000', 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '6.7.8.9', 'port': '10', 'path': '/a/c', 'device': 'sdf1', 'partition': '30', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, 'x-put-timestamp': '0000012345.00000', 'referer': 'PUT http://localhost/sda1/p/a/c', 'user-agent': 'container-server %d' % os.getpid(), 'x-trans-id': '-'})}) def test_serv_reserv(self): # Test replication_server flag was set from configuration file. container_controller = container_server.ContainerController conf = {'devices': self.testdir, 'mount_check': 'false'} self.assertEquals(container_controller(conf).replication_server, None) for val in [True, '1', 'True', 'true']: conf['replication_server'] = val self.assertTrue(container_controller(conf).replication_server) for val in [False, 0, '0', 'False', 'false', 'test_string']: conf['replication_server'] = val self.assertFalse(container_controller(conf).replication_server) def test_list_allowed_methods(self): # Test list of allowed_methods obj_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'POST'] repl_methods = ['REPLICATE'] for method_name in obj_methods: method = getattr(self.controller, method_name) self.assertFalse(hasattr(method, 'replication')) for method_name in repl_methods: method = getattr(self.controller, method_name) self.assertEquals(method.replication, True) def test_correct_allowed_method(self): # Test correct work for allowed method using # swift.container.server.ContainerController.__call__ inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.controller = container_server.ContainerController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false'}) def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} method_res = mock.MagicMock() mock_method = public(lambda x: mock.MagicMock(return_value=method_res)) with mock.patch.object(self.controller, method, new=mock_method): response = self.controller.__call__(env, start_response) self.assertEqual(response, method_res) def test_not_allowed_method(self): # Test correct work for NOT allowed method using # swift.container.server.ContainerController.__call__ inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.controller = container_server.ContainerController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false'}) def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} answer = ['

Method Not Allowed

The method is not ' 'allowed for this resource.

'] mock_method = replication(public(lambda x: mock.MagicMock())) with mock.patch.object(self.controller, method, new=mock_method): response = self.controller.__call__(env, start_response) self.assertEqual(response, answer) def test_GET_log_requests_true(self): self.controller.logger = FakeLogger() self.controller.log_requests = True req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertTrue(self.controller.logger.log_dict['info']) def test_GET_log_requests_false(self): self.controller.logger = FakeLogger() self.controller.log_requests = False req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 404) self.assertFalse(self.controller.logger.log_dict['info']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/container/test_replicator.py0000664000175400017540000000455112323703611023567 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from swift.container import replicator from swift.common.utils import normalize_timestamp class TestReplicator(unittest.TestCase): def setUp(self): self.orig_ring = replicator.db_replicator.ring.Ring replicator.db_replicator.ring.Ring = lambda *args, **kwargs: None def tearDown(self): replicator.db_replicator.ring.Ring = self.orig_ring def test_report_up_to_date(self): repl = replicator.ContainerReplicator({}) info = {'put_timestamp': normalize_timestamp(1), 'delete_timestamp': normalize_timestamp(0), 'object_count': 0, 'bytes_used': 0, 'reported_put_timestamp': normalize_timestamp(1), 'reported_delete_timestamp': normalize_timestamp(0), 'reported_object_count': 0, 'reported_bytes_used': 0} self.assertTrue(repl.report_up_to_date(info)) info['delete_timestamp'] = normalize_timestamp(2) self.assertFalse(repl.report_up_to_date(info)) info['reported_delete_timestamp'] = normalize_timestamp(2) self.assertTrue(repl.report_up_to_date(info)) info['object_count'] = 1 self.assertFalse(repl.report_up_to_date(info)) info['reported_object_count'] = 1 self.assertTrue(repl.report_up_to_date(info)) info['bytes_used'] = 1 self.assertFalse(repl.report_up_to_date(info)) info['reported_bytes_used'] = 1 self.assertTrue(repl.report_up_to_date(info)) info['put_timestamp'] = normalize_timestamp(3) self.assertFalse(repl.report_up_to_date(info)) info['reported_put_timestamp'] = normalize_timestamp(3) self.assertTrue(repl.report_up_to_date(info)) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/container/test_auditor.py0000664000175400017540000001014312323703611023064 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import mock import time import os import random from tempfile import mkdtemp from shutil import rmtree from swift.container import auditor from test.unit import FakeLogger class FakeContainerBroker(object): def __init__(self, path): self.path = path self.db_file = path self.file = os.path.basename(path) def is_deleted(self): return False def get_info(self): if self.file.startswith('fail'): raise ValueError if self.file.startswith('true'): return 'ok' class TestAuditor(unittest.TestCase): def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_auditor') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) fnames = ['true1.db', 'true2.db', 'true3.db', 'fail1.db', 'fail2.db'] for fn in fnames: with open(os.path.join(self.testdir, fn), 'w+') as f: f.write(' ') def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) @mock.patch('swift.container.auditor.ContainerBroker', FakeContainerBroker) def test_run_forever(self): sleep_times = random.randint(5, 10) call_times = sleep_times - 1 class FakeTime(object): def __init__(self): self.times = 0 def sleep(self, sec): self.times += 1 if self.times < sleep_times: time.sleep(0.1) else: # stop forever by an error raise ValueError() def time(self): return time.time() conf = {} test_auditor = auditor.ContainerAuditor(conf) with mock.patch('swift.container.auditor.time', FakeTime()): def fake_audit_location_generator(*args, **kwargs): files = os.listdir(self.testdir) return [(os.path.join(self.testdir, f), '', '') for f in files] with mock.patch('swift.container.auditor.audit_location_generator', fake_audit_location_generator): self.assertRaises(ValueError, test_auditor.run_forever) self.assertEquals(test_auditor.container_failures, 2 * call_times) self.assertEquals(test_auditor.container_passes, 3 * call_times) @mock.patch('swift.container.auditor.ContainerBroker', FakeContainerBroker) def test_run_once(self): conf = {} test_auditor = auditor.ContainerAuditor(conf) def fake_audit_location_generator(*args, **kwargs): files = os.listdir(self.testdir) return [(os.path.join(self.testdir, f), '', '') for f in files] with mock.patch('swift.container.auditor.audit_location_generator', fake_audit_location_generator): test_auditor.run_once() self.assertEquals(test_auditor.container_failures, 2) self.assertEquals(test_auditor.container_passes, 3) @mock.patch('swift.container.auditor.ContainerBroker', FakeContainerBroker) def test_container_auditor(self): conf = {} test_auditor = auditor.ContainerAuditor(conf) files = os.listdir(self.testdir) for f in files: path = os.path.join(self.testdir, f) test_auditor.container_audit(path) self.assertEquals(test_auditor.container_failures, 2) self.assertEquals(test_auditor.container_passes, 3) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/container/test_updater.py0000664000175400017540000002024012323703611023060 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import cPickle as pickle import os import unittest from contextlib import closing from gzip import GzipFile from shutil import rmtree from tempfile import mkdtemp from eventlet import spawn, Timeout, listen from swift.common import utils from swift.container import updater as container_updater from swift.container.backend import ContainerBroker, DATADIR from swift.common.ring import RingData from swift.common.utils import normalize_timestamp class TestContainerUpdater(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' self.testdir = os.path.join(mkdtemp(), 'tmp_test_container_updater') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) ring_file = os.path.join(self.testdir, 'account.ring.gz') with closing(GzipFile(ring_file, 'wb')) as f: pickle.dump( RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', 'zone': 0}, {'id': 1, 'ip': '127.0.0.1', 'port': 12345, 'device': 'sda1', 'zone': 2}], 30), f) self.devices_dir = os.path.join(self.testdir, 'devices') os.mkdir(self.devices_dir) self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1) def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) def test_creation(self): cu = container_updater.ContainerUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '2', 'node_timeout': '5', }) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) self.assertEquals(cu.interval, 1) self.assertEquals(cu.concurrency, 2) self.assertEquals(cu.node_timeout, 5) self.assert_(cu.get_account_ring() is not None) def test_run_once(self): cu = container_updater.ContainerUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15', 'account_suppression_time': 0 }) cu.run_once() containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) cu.run_once() self.assert_(os.path.exists(containers_dir)) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', container='c') cb.initialize(normalize_timestamp(1)) cu.run_once() info = cb.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') cu.run_once() info = cb.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 3) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) def accept(sock, addr, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/0/a/c HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assert_('x-put-timestamp' in headers) self.assert_('x-delete-timestamp' in headers) self.assert_('x-object-count' in headers) self.assert_('x-bytes-used' in headers) except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen(('127.0.0.1', 0)) def spawn_accepts(): events = [] for _junk in xrange(2): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr, 201)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 3) self.assertEquals(info['reported_object_count'], 1) self.assertEquals(info['reported_bytes_used'], 3) def test_unicode(self): cu = container_updater.ContainerUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15', }) containers_dir = os.path.join(self.sda1, DATADIR) os.mkdir(containers_dir) subdir = os.path.join(containers_dir, 'subdir') os.mkdir(subdir) cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', container='\xce\xa9') cb.initialize(normalize_timestamp(1)) cb.put_object('\xce\xa9', normalize_timestamp(2), 3, 'text/plain', '68b329da9893e34099c7d8ad5cb9c940') def accept(sock, addr): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 201 OK\r\nContent-Length: 0\r\n\r\n') out.flush() inc.read() except BaseException as err: import traceback traceback.print_exc() return err return None bindsock = listen(('127.0.0.1', 0)) def spawn_accepts(): events = [] for _junk in xrange(2): with Timeout(3): sock, addr = bindsock.accept() events.append(spawn(accept, sock, addr)) return events spawned = spawn(spawn_accepts) for dev in cu.get_account_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.run_once() for event in spawned.wait(): err = event.wait() if err: raise err info = cb.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 3) self.assertEquals(info['reported_object_count'], 1) self.assertEquals(info['reported_bytes_used'], 3) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/container/test_sync.py0000664000175400017540000012015412323703614022400 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import re import unittest from contextlib import nested import mock from test.unit import FakeLogger from swift.container import sync from swift.common import utils from swift.common.exceptions import ClientException utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'endcap' class FakeRing(object): def __init__(self): self.devs = [{'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda'} for x in xrange(3)] def get_nodes(self, account, container=None, obj=None): return 1, list(self.devs) class FakeContainerBroker(object): def __init__(self, path, metadata=None, info=None, deleted=False, items_since=None): self.db_file = path self.metadata = metadata if metadata else {} self.info = info if info else {} self.deleted = deleted self.items_since = items_since if items_since else [] self.sync_point1 = -1 self.sync_point2 = -1 def get_info(self): return self.info def is_deleted(self): return self.deleted def get_items_since(self, sync_point, limit): if sync_point < 0: sync_point = 0 return self.items_since[sync_point:sync_point + limit] def set_x_container_sync_points(self, sync_point1, sync_point2): self.sync_point1 = sync_point1 self.sync_point2 = sync_point2 class TestContainerSync(unittest.TestCase): def test_FileLikeIter(self): # Retained test to show new FileLikeIter acts just like the removed # _Iter2FileLikeObject did. flo = sync.FileLikeIter(iter(['123', '4567', '89', '0'])) expect = '1234567890' got = flo.read(2) self.assertTrue(len(got) <= 2) self.assertEquals(got, expect[:len(got)]) expect = expect[len(got):] got = flo.read(5) self.assertTrue(len(got) <= 5) self.assertEquals(got, expect[:len(got)]) expect = expect[len(got):] self.assertEquals(flo.read(), expect) self.assertEquals(flo.read(), '') self.assertEquals(flo.read(2), '') flo = sync.FileLikeIter(iter(['123', '4567', '89', '0'])) self.assertEquals(flo.read(), '1234567890') self.assertEquals(flo.read(), '') self.assertEquals(flo.read(2), '') def test_init(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) self.assertTrue(cs.container_ring is cring) self.assertTrue(cs.object_ring is oring) def test_run_forever(self): # This runs runs_forever with fakes to succeed for two loops, the first # causing a report but no interval sleep, the second no report but an # interval sleep. time_calls = [0] sleep_calls = [] audit_location_generator_calls = [0] def fake_time(): time_calls[0] += 1 returns = [1, # Initialized reported time 1, # Start time 3602, # Is it report time (yes) 3602, # Report time 3602, # Elapsed time for "under interval" (no) 3602, # Start time 3603, # Is it report time (no) 3603] # Elapsed time for "under interval" (yes) if time_calls[0] == len(returns) + 1: raise Exception('we are now done') return returns[time_calls[0] - 1] def fake_sleep(amount): sleep_calls.append(amount) def fake_audit_location_generator(*args, **kwargs): audit_location_generator_calls[0] += 1 # Makes .container_sync() short-circuit yield 'container.db', 'device', 'partition' return orig_time = sync.time orig_sleep = sync.sleep orig_ContainerBroker = sync.ContainerBroker orig_audit_location_generator = sync.audit_location_generator try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c'}) sync.time = fake_time sync.sleep = fake_sleep cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) sync.audit_location_generator = fake_audit_location_generator cs.run_forever(1, 2, a=3, b=4, verbose=True) except Exception as err: if str(err) != 'we are now done': raise finally: sync.time = orig_time sync.sleep = orig_sleep sync.audit_location_generator = orig_audit_location_generator sync.ContainerBroker = orig_ContainerBroker self.assertEquals(time_calls, [9]) self.assertEquals(len(sleep_calls), 2) self.assertTrue(sleep_calls[0] <= cs.interval) self.assertTrue(sleep_calls[1] == cs.interval - 1) self.assertEquals(audit_location_generator_calls, [2]) self.assertEquals(cs.reported, 3602) def test_run_once(self): # This runs runs_once with fakes twice, the first causing an interim # report, the second with no interim report. time_calls = [0] audit_location_generator_calls = [0] def fake_time(): time_calls[0] += 1 returns = [1, # Initialized reported time 1, # Start time 3602, # Is it report time (yes) 3602, # Report time 3602, # End report time 3602, # For elapsed 3602, # Start time 3603, # Is it report time (no) 3604, # End report time 3605] # For elapsed if time_calls[0] == len(returns) + 1: raise Exception('we are now done') return returns[time_calls[0] - 1] def fake_audit_location_generator(*args, **kwargs): audit_location_generator_calls[0] += 1 # Makes .container_sync() short-circuit yield 'container.db', 'device', 'partition' return orig_time = sync.time orig_audit_location_generator = sync.audit_location_generator orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c'}) sync.time = fake_time cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) sync.audit_location_generator = fake_audit_location_generator cs.run_once(1, 2, a=3, b=4, verbose=True) self.assertEquals(time_calls, [6]) self.assertEquals(audit_location_generator_calls, [1]) self.assertEquals(cs.reported, 3602) cs.run_once() except Exception as err: if str(err) != 'we are now done': raise finally: sync.time = orig_time sync.audit_location_generator = orig_audit_location_generator sync.ContainerBroker = orig_ContainerBroker self.assertEquals(time_calls, [10]) self.assertEquals(audit_location_generator_calls, [2]) self.assertEquals(cs.reported, 3604) def test_container_sync_not_db(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) self.assertEquals(cs.container_failures, 0) def test_container_sync_missing_db(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) def test_container_sync_not_my_db(self): # Db could be there due to handoff replication so test that we ignore # those. cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c'}) cs._myips = ['127.0.0.1'] # No match cs._myport = 1 # No match cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) cs._myips = ['10.0.0.0'] # Match cs._myport = 1 # No match cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) cs._myips = ['127.0.0.1'] # No match cs._myport = 1000 # Match cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match # This complete match will cause the 1 container failure since the # broker's info doesn't contain sync point keys cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) finally: sync.ContainerBroker = orig_ContainerBroker def test_container_sync_deleted(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c'}, deleted=False) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match # This complete match will cause the 1 container failure since the # broker's info doesn't contain sync point keys cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c'}, deleted=True) # This complete match will not cause any more container failures # since the broker indicates deletion cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) finally: sync.ContainerBroker = orig_ContainerBroker def test_container_sync_no_to_or_key(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) orig_ContainerBroker = sync.ContainerBroker try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match # This complete match will be skipped since the broker's metadata # has no x-container-sync-to or x-container-sync-key cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) self.assertEquals(cs.container_skips, 1) sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1)}) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match # This complete match will be skipped since the broker's metadata # has no x-container-sync-key cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) self.assertEquals(cs.container_skips, 2) sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-key': ('key', 1)}) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match # This complete match will be skipped since the broker's metadata # has no x-container-sync-to cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 0) self.assertEquals(cs.container_skips, 3) sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = [] # This complete match will cause a container failure since the # sync-to won't validate as allowed. cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 3) sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] # This complete match will succeed completely since the broker # get_items_since will return no new rows. cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 3) finally: sync.ContainerBroker = orig_ContainerBroker def test_container_stop_at(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) orig_ContainerBroker = sync.ContainerBroker orig_time = sync.time try: sync.ContainerBroker = lambda p: FakeContainerBroker( p, info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=['erroneous data']) cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] # This sync will fail since the items_since data is bad. cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) # Set up fake times to make the sync short-circuit as having taken # too long fake_times = [ 1.0, # Compute the time to move on 100000.0, # Compute if it's time to move on from first loop 100000.0] # Compute if it's time to move on from second loop def fake_time(): return fake_times.pop(0) sync.time = fake_time # This same sync won't fail since it will look like it took so long # as to be time to move on (before it ever actually tries to do # anything). cs.container_sync('isa.db') self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) finally: sync.ContainerBroker = orig_ContainerBroker sync.time = orig_time def test_container_first_loop(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) def fake_hash_path(account, container, obj, raw_digest=False): # Ensures that no rows match for full syncing, ordinal is 0 and # all hashes are 0 return '\x00' * 16 fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': 2, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o'}]) with nested( mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb), mock.patch('swift.container.sync.hash_path', fake_hash_path)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Succeeds because no rows match self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) def fake_hash_path(account, container, obj, raw_digest=False): # Ensures that all rows match for full syncing, ordinal is 0 # and all hashes are 1 return '\x01' * 16 fcb = FakeContainerBroker('path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': 1, 'x_container_sync_point2': 1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o'}]) with nested( mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb), mock.patch('swift.container.sync.hash_path', fake_hash_path)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Succeeds because the two sync points haven't deviated yet self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, -1) self.assertEquals(fcb.sync_point2, -1) fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': 2, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o'}]) with mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Fails because container_sync_row will fail since the row has no # 'deleted' key self.assertEquals(cs.container_failures, 2) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) def fake_delete_object(*args, **kwargs): raise ClientException fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': 2, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', 'deleted': True}]) with nested( mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb), mock.patch('swift.container.sync.delete_object', fake_delete_object)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Fails because delete_object fails self.assertEquals(cs.container_failures, 3) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, -1) fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': 2, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', 'deleted': True}]) with nested( mock.patch('swift.container.sync.ContainerBroker', lambda p: fcb), mock.patch('swift.container.sync.delete_object', lambda *x, **y: None)): cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Succeeds because delete_object succeeds self.assertEquals(cs.container_failures, 3) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, None) self.assertEquals(fcb.sync_point2, 1) def test_container_second_loop(self): cring = FakeRing() oring = FakeRing() cs = sync.ContainerSync({}, container_ring=cring, object_ring=oring) orig_ContainerBroker = sync.ContainerBroker orig_hash_path = sync.hash_path orig_delete_object = sync.delete_object try: # We'll ensure the first loop is always skipped by keeping the two # sync points equal def fake_hash_path(account, container, obj, raw_digest=False): # Ensures that no rows match for second loop, ordinal is 0 and # all hashes are 1 return '\x01' * 16 sync.hash_path = fake_hash_path fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o'}]) sync.ContainerBroker = lambda p: fcb cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Succeeds because no rows match self.assertEquals(cs.container_failures, 0) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, 1) self.assertEquals(fcb.sync_point2, None) def fake_hash_path(account, container, obj, raw_digest=False): # Ensures that all rows match for second loop, ordinal is 0 and # all hashes are 0 return '\x00' * 16 def fake_delete_object(*args, **kwargs): pass sync.hash_path = fake_hash_path sync.delete_object = fake_delete_object fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o'}]) sync.ContainerBroker = lambda p: fcb cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Fails because row is missing 'deleted' key # Nevertheless the fault is skipped self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, 1) self.assertEquals(fcb.sync_point2, None) fcb = FakeContainerBroker( 'path', info={'account': 'a', 'container': 'c', 'x_container_sync_point1': -1, 'x_container_sync_point2': -1}, metadata={'x-container-sync-to': ('http://127.0.0.1/a/c', 1), 'x-container-sync-key': ('key', 1)}, items_since=[{'ROWID': 1, 'name': 'o', 'created_at': '1.2', 'deleted': True}]) sync.ContainerBroker = lambda p: fcb cs._myips = ['10.0.0.0'] # Match cs._myport = 1000 # Match cs.allowed_sync_hosts = ['127.0.0.1'] cs.container_sync('isa.db') # Succeeds because row now has 'deleted' key and delete_object # succeeds self.assertEquals(cs.container_failures, 1) self.assertEquals(cs.container_skips, 0) self.assertEquals(fcb.sync_point1, 1) self.assertEquals(fcb.sync_point2, None) finally: sync.ContainerBroker = orig_ContainerBroker sync.hash_path = orig_hash_path sync.delete_object = orig_delete_object def test_container_sync_row_delete(self): self._test_container_sync_row_delete(None, None) def test_container_sync_row_delete_using_realms(self): self._test_container_sync_row_delete('US', 'realm_key') def _test_container_sync_row_delete(self, realm, realm_key): orig_uuid = sync.uuid orig_delete_object = sync.delete_object try: class FakeUUID(object): class uuid4(object): hex = 'abcdef' sync.uuid = FakeUUID def fake_delete_object(path, name=None, headers=None, proxy=None): self.assertEquals(path, 'http://sync/to/path') self.assertEquals(name, 'object') if realm: self.assertEquals(headers, { 'x-container-sync-auth': 'US abcdef 90e95aabb45a6cdc0892a3db5535e7f918428c90', 'x-timestamp': '1.2'}) else: self.assertEquals( headers, {'x-container-sync-key': 'key', 'x-timestamp': '1.2'}) self.assertEquals(proxy, 'http://proxy') sync.delete_object = fake_delete_object cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) cs.http_proxies = ['http://proxy'] # Success self.assertTrue(cs.container_sync_row( {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), 'info', realm, realm_key)) self.assertEquals(cs.container_deletes, 1) exc = [] def fake_delete_object(path, name=None, headers=None, proxy=None): exc.append(Exception('test exception')) raise exc[-1] sync.delete_object = fake_delete_object # Failure because of delete_object exception self.assertFalse(cs.container_sync_row( {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), 'info', realm, realm_key)) self.assertEquals(cs.container_deletes, 1) self.assertEquals(len(exc), 1) self.assertEquals(str(exc[-1]), 'test exception') def fake_delete_object(path, name=None, headers=None, proxy=None): exc.append(ClientException('test client exception')) raise exc[-1] sync.delete_object = fake_delete_object # Failure because of delete_object exception self.assertFalse(cs.container_sync_row( {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), 'info', realm, realm_key)) self.assertEquals(cs.container_deletes, 1) self.assertEquals(len(exc), 2) self.assertEquals(str(exc[-1]), 'test client exception') def fake_delete_object(path, name=None, headers=None, proxy=None): exc.append(ClientException('test client exception', http_status=404)) raise exc[-1] sync.delete_object = fake_delete_object # Success because the object wasn't even found self.assertTrue(cs.container_sync_row( {'deleted': True, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), 'info', realm, realm_key)) self.assertEquals(cs.container_deletes, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test client exception: 404') finally: sync.uuid = orig_uuid sync.delete_object = orig_delete_object def test_container_sync_row_put(self): self._test_container_sync_row_put(None, None) def test_container_sync_row_put_using_realms(self): self._test_container_sync_row_put('US', 'realm_key') def _test_container_sync_row_put(self, realm, realm_key): orig_uuid = sync.uuid orig_shuffle = sync.shuffle orig_put_object = sync.put_object orig_direct_get_object = sync.direct_get_object try: class FakeUUID(object): class uuid4(object): hex = 'abcdef' sync.uuid = FakeUUID sync.shuffle = lambda x: x def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None): self.assertEquals(sync_to, 'http://sync/to/path') self.assertEquals(name, 'object') if realm: self.assertEqual(headers, { 'x-container-sync-auth': 'US abcdef ef62c64bb88a33fa00722daa23d5d43253164962', 'x-timestamp': '1.2', 'etag': 'etagvalue', 'other-header': 'other header value'}) else: self.assertEquals(headers, { 'x-container-sync-key': 'key', 'x-timestamp': '1.2', 'other-header': 'other header value', 'etag': 'etagvalue'}) self.assertEquals(contents.read(), 'contents') self.assertEquals(proxy, 'http://proxy') sync.put_object = fake_put_object cs = sync.ContainerSync({}, container_ring=FakeRing(), object_ring=FakeRing()) cs.http_proxies = ['http://proxy'] def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): return ({'other-header': 'other header value', 'etag': '"etagvalue"', 'x-timestamp': '1.2'}, iter('contents')) sync.direct_get_object = fake_direct_get_object # Success as everything says it worked self.assertTrue(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 1) def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): return ({'date': 'date value', 'last-modified': 'last modified value', 'x-timestamp': '1.2', 'other-header': 'other header value', 'etag': '"etagvalue"'}, iter('contents')) sync.direct_get_object = fake_direct_get_object # Success as everything says it worked, also checks 'date' and # 'last-modified' headers are removed and that 'etag' header is # stripped of double quotes. self.assertTrue(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) exc = [] def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): exc.append(Exception('test exception')) raise exc[-1] sync.direct_get_object = fake_direct_get_object # Fail due to completely unexpected exception self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test exception') exc = [] def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): exc.append(ClientException('test client exception')) raise exc[-1] sync.direct_get_object = fake_direct_get_object # Fail due to all direct_get_object calls failing self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertEquals(len(exc), 3) self.assertEquals(str(exc[-1]), 'test client exception') def fake_direct_get_object(node, part, account, container, obj, resp_chunk_size=1): return ({'other-header': 'other header value', 'x-timestamp': '1.2', 'etag': '"etagvalue"'}, iter('contents')) def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None): raise ClientException('test client exception', http_status=401) sync.direct_get_object = fake_direct_get_object sync.put_object = fake_put_object cs.logger = FakeLogger() # Fail due to 401 self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assert_(re.match('Unauth ', cs.logger.log_dict['info'][0][0][0])) def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None): raise ClientException('test client exception', http_status=404) sync.put_object = fake_put_object # Fail due to 404 cs.logger = FakeLogger() self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assert_(re.match('Not found ', cs.logger.log_dict['info'][0][0][0])) def fake_put_object(sync_to, name=None, headers=None, contents=None, proxy=None): raise ClientException('test client exception', http_status=503) sync.put_object = fake_put_object # Fail due to 503 self.assertFalse(cs.container_sync_row( {'deleted': False, 'name': 'object', 'created_at': '1.2'}, 'http://sync/to/path', 'key', FakeContainerBroker('broker'), { 'account': 'a', 'container': 'c'}, realm, realm_key)) self.assertEquals(cs.container_puts, 2) self.assertTrue( cs.logger.log_dict['exception'][0][0][0].startswith( 'ERROR Syncing ')) finally: sync.uuid = orig_uuid sync.shuffle = orig_shuffle sync.put_object = orig_put_object sync.direct_get_object = orig_direct_get_object def test_select_http_proxy_None(self): cs = sync.ContainerSync( {'sync_proxy': ''}, container_ring=FakeRing(), object_ring=FakeRing()) self.assertEqual(cs.select_http_proxy(), None) def test_select_http_proxy_one(self): cs = sync.ContainerSync( {'sync_proxy': 'http://one'}, container_ring=FakeRing(), object_ring=FakeRing()) self.assertEqual(cs.select_http_proxy(), 'http://one') def test_select_http_proxy_multiple(self): cs = sync.ContainerSync( {'sync_proxy': 'http://one,http://two,http://three'}, container_ring=FakeRing(), object_ring=FakeRing()) self.assertEqual( set(cs.http_proxies), set(['http://one', 'http://two', 'http://three'])) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/container/test_backend.py0000664000175400017540000016021612323703611023013 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests for swift.container.backend """ import hashlib import unittest from time import sleep, time from uuid import uuid4 from swift.container.backend import ContainerBroker from swift.common.utils import normalize_timestamp class TestContainerBroker(unittest.TestCase): """Tests for ContainerBroker""" def test_creation(self): # Test ContainerBroker.__init__ broker = ContainerBroker(':memory:', account='a', container='c') self.assertEqual(broker.db_file, ':memory:') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: curs = conn.cursor() curs.execute('SELECT 1') self.assertEqual(curs.fetchall()[0][0], 1) def test_exception(self): # Test ContainerBroker throwing a conn away after # unhandled exception first_conn = None broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: first_conn = conn try: with broker.get() as conn: self.assertEquals(first_conn, conn) raise Exception('OMG') except Exception: pass self.assert_(broker.conn is None) def test_empty(self): # Test ContainerBroker.empty broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) self.assert_(broker.empty()) broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') self.assert_(not broker.empty()) sleep(.00001) broker.delete_object('o', normalize_timestamp(time())) self.assert_(broker.empty()) def test_reclaim(self): broker = ContainerBroker(':memory:', account='test_account', container='test_container') broker.initialize(normalize_timestamp('1')) broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 1) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 1) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.delete_object('o', normalize_timestamp(time())) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) broker.reclaim(normalize_timestamp(time() - 999), time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) sleep(.00001) broker.reclaim(normalize_timestamp(time()), time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) # Test the return values of reclaim() broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') # Test before deletion broker.reclaim(normalize_timestamp(time()), time()) broker.delete_db(normalize_timestamp(time())) def test_delete_object(self): # Test ContainerBroker.delete_object broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 1) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 0) sleep(.00001) broker.delete_object('o', normalize_timestamp(time())) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 0").fetchone()[0], 0) self.assertEquals(conn.execute( "SELECT count(*) FROM object " "WHERE deleted = 1").fetchone()[0], 1) def test_put_object(self): # Test ContainerBroker.put_object broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) # Create initial object timestamp = normalize_timestamp(time()) broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 123) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], '5af83e3196bf99f440f31f2e1a6c9afe') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Reput same event broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 123) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], '5af83e3196bf99f440f31f2e1a6c9afe') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_object('"{}"', timestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 124) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], 'aa0749bacbc79ec65fe206943d8fe449') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Put old event otimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_object('"{}"', otimestamp, 124, 'application/x-test', 'aa0749bacbc79ec65fe206943d8fe449') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 124) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], 'aa0749bacbc79ec65fe206943d8fe449') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Put old delete event dtimestamp = normalize_timestamp(float(timestamp) - 1) broker.put_object('"{}"', dtimestamp, 0, '', '', deleted=1) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 124) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], 'aa0749bacbc79ec65fe206943d8fe449') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Put new delete event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_object('"{}"', timestamp, 0, '', '', deleted=1) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 1) # Put new event sleep(.00001) timestamp = normalize_timestamp(time()) broker.put_object('"{}"', timestamp, 123, 'application/x-test', '5af83e3196bf99f440f31f2e1a6c9afe') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 123) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], '5af83e3196bf99f440f31f2e1a6c9afe') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # We'll use this later sleep(.0001) in_between_timestamp = normalize_timestamp(time()) # New post event sleep(.0001) previous_timestamp = timestamp timestamp = normalize_timestamp(time()) with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], previous_timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 123) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], '5af83e3196bf99f440f31f2e1a6c9afe') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) # Put event from after last put but before last post timestamp = in_between_timestamp broker.put_object('"{}"', timestamp, 456, 'application/x-test3', '6af83e3196bf99f440f31f2e1a6c9afe') with broker.get() as conn: self.assertEquals(conn.execute( "SELECT name FROM object").fetchone()[0], '"{}"') self.assertEquals(conn.execute( "SELECT created_at FROM object").fetchone()[0], timestamp) self.assertEquals(conn.execute( "SELECT size FROM object").fetchone()[0], 456) self.assertEquals(conn.execute( "SELECT content_type FROM object").fetchone()[0], 'application/x-test3') self.assertEquals(conn.execute( "SELECT etag FROM object").fetchone()[0], '6af83e3196bf99f440f31f2e1a6c9afe') self.assertEquals(conn.execute( "SELECT deleted FROM object").fetchone()[0], 0) def test_get_info(self): # Test ContainerBroker.get_info broker = ContainerBroker(':memory:', account='test1', container='test2') broker.initialize(normalize_timestamp('1')) info = broker.get_info() self.assertEquals(info['account'], 'test1') self.assertEquals(info['container'], 'test2') self.assertEquals(info['hash'], '00000000000000000000000000000000') info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 123) sleep(.00001) broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 246) sleep(.00001) broker.put_object('o2', normalize_timestamp(time()), 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 1123) sleep(.00001) broker.delete_object('o1', normalize_timestamp(time())) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) sleep(.00001) broker.delete_object('o2', normalize_timestamp(time())) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) info = broker.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) def test_set_x_syncs(self): broker = ContainerBroker(':memory:', account='test1', container='test2') broker.initialize(normalize_timestamp('1')) info = broker.get_info() self.assertEquals(info['x_container_sync_point1'], -1) self.assertEquals(info['x_container_sync_point2'], -1) broker.set_x_container_sync_points(1, 2) info = broker.get_info() self.assertEquals(info['x_container_sync_point1'], 1) self.assertEquals(info['x_container_sync_point2'], 2) def test_get_report_info(self): broker = ContainerBroker(':memory:', account='test1', container='test2') broker.initialize(normalize_timestamp('1')) info = broker.get_info() self.assertEquals(info['account'], 'test1') self.assertEquals(info['container'], 'test2') self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 123) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 246) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) sleep(.00001) broker.put_object('o2', normalize_timestamp(time()), 1000, 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe') info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 1123) self.assertEquals(info['reported_object_count'], 0) self.assertEquals(info['reported_bytes_used'], 0) put_timestamp = normalize_timestamp(time()) sleep(.001) delete_timestamp = normalize_timestamp(time()) broker.reported(put_timestamp, delete_timestamp, 2, 1123) info = broker.get_info() self.assertEquals(info['object_count'], 2) self.assertEquals(info['bytes_used'], 1123) self.assertEquals(info['reported_put_timestamp'], put_timestamp) self.assertEquals(info['reported_delete_timestamp'], delete_timestamp) self.assertEquals(info['reported_object_count'], 2) self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) broker.delete_object('o1', normalize_timestamp(time())) info = broker.get_info() self.assertEquals(info['object_count'], 1) self.assertEquals(info['bytes_used'], 1000) self.assertEquals(info['reported_object_count'], 2) self.assertEquals(info['reported_bytes_used'], 1123) sleep(.00001) broker.delete_object('o2', normalize_timestamp(time())) info = broker.get_info() self.assertEquals(info['object_count'], 0) self.assertEquals(info['bytes_used'], 0) self.assertEquals(info['reported_object_count'], 2) self.assertEquals(info['reported_bytes_used'], 1123) def test_list_objects_iter(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d/%04d' % (obj1, obj2), normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2/0051/%04d' % obj, normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3/%04d/0049' % obj, normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0099') listing = broker.list_objects_iter(100, '', '0/0050', None, '') self.assertEquals(len(listing), 50) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0049') listing = broker.list_objects_iter(100, '0/0099', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '1/0074') listing = broker.list_objects_iter(55, '1/0074', None, None, '') self.assertEquals(len(listing), 55) self.assertEquals(listing[0][0], '1/0075') self.assertEquals(listing[-1][0], '2/0004') listing = broker.list_objects_iter(10, '', None, '0/01', '') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '0/0109') listing = broker.list_objects_iter(10, '', None, '0/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0009') # Same as above, but using the path argument. listing = broker.list_objects_iter(10, '', None, None, '', '0') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0009') listing = broker.list_objects_iter(10, '', None, '', '/') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['0/', '1/', '2/', '3/']) listing = broker.list_objects_iter(10, '2', None, None, '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['2/', '3/']) listing = broker.list_objects_iter(10, '2/', None, None, '/') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3/']) listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '2/0051') self.assertEquals(listing[1][0], '2/0051/') self.assertEquals(listing[2][0], '2/0052') self.assertEquals(listing[-1][0], '2/0059') listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0045/', '3/0046', '3/0046/', '3/0047', '3/0047/', '3/0048', '3/0048/', '3/0049', '3/0049/', '3/0050']) broker.put_object('3/0049/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3/0048', None, None, None) self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', '3/0052', '3/0052/0049']) listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['3/0048/', '3/0049', '3/0049/', '3/0050', '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) listing = broker.list_objects_iter(10, None, None, '3/0049/', '/') self.assertEquals(len(listing), 2) self.assertEquals( [row[0] for row in listing], ['3/0049/', '3/0049/0049']) listing = broker.list_objects_iter(10, None, None, None, None, '3/0049') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3/0049/0049']) listing = broker.list_objects_iter(2, None, None, '3/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) listing = broker.list_objects_iter(2, None, None, None, None, '3') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) def test_list_objects_iter_non_slash(self): # Test ContainerBroker.list_objects_iter using a # delimiter that is not a slash broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) for obj1 in xrange(4): for obj2 in xrange(125): broker.put_object('%d:%04d' % (obj1, obj2), normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('2:0051:%04d' % obj, normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') for obj in xrange(125): broker.put_object('3:%04d:0049' % obj, normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(100, '', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0:0000') self.assertEquals(listing[-1][0], '0:0099') listing = broker.list_objects_iter(100, '', '0:0050', None, '') self.assertEquals(len(listing), 50) self.assertEquals(listing[0][0], '0:0000') self.assertEquals(listing[-1][0], '0:0049') listing = broker.list_objects_iter(100, '0:0099', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0:0100') self.assertEquals(listing[-1][0], '1:0074') listing = broker.list_objects_iter(55, '1:0074', None, None, '') self.assertEquals(len(listing), 55) self.assertEquals(listing[0][0], '1:0075') self.assertEquals(listing[-1][0], '2:0004') listing = broker.list_objects_iter(10, '', None, '0:01', '') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0:0100') self.assertEquals(listing[-1][0], '0:0109') listing = broker.list_objects_iter(10, '', None, '0:', ':') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0:0000') self.assertEquals(listing[-1][0], '0:0009') # Same as above, but using the path argument, so nothing should be # returned since path uses a '/' as a delimiter. listing = broker.list_objects_iter(10, '', None, None, '', '0') self.assertEquals(len(listing), 0) listing = broker.list_objects_iter(10, '', None, '', ':') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['0:', '1:', '2:', '3:']) listing = broker.list_objects_iter(10, '2', None, None, ':') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['2:', '3:']) listing = broker.list_objects_iter(10, '2:', None, None, ':') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3:']) listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '2:0051') self.assertEquals(listing[1][0], '2:0051:') self.assertEquals(listing[2][0], '2:0052') self.assertEquals(listing[-1][0], '2:0059') listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3:0045:', '3:0046', '3:0046:', '3:0047', '3:0047:', '3:0048', '3:0048:', '3:0049', '3:0049:', '3:0050']) broker.put_object('3:0049:', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(10, '3:0048', None, None, None) self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['3:0048:0049', '3:0049', '3:0049:', '3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049', '3:0052', '3:0052:0049']) listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':') self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['3:0048:', '3:0049', '3:0049:', '3:0050', '3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053']) listing = broker.list_objects_iter(10, None, None, '3:0049:', ':') self.assertEquals(len(listing), 2) self.assertEquals( [row[0] for row in listing], ['3:0049:', '3:0049:0049']) # Same as above, but using the path argument, so nothing should be # returned since path uses a '/' as a delimiter. listing = broker.list_objects_iter(10, None, None, None, None, '3:0049') self.assertEquals(len(listing), 0) listing = broker.list_objects_iter(2, None, None, '3:', ':') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:']) listing = broker.list_objects_iter(2, None, None, None, None, '3') self.assertEquals(len(listing), 0) def test_list_objects_iter_prefix_delim(self): # Test ContainerBroker.list_objects_iter broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object( '/pets/dogs/1', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( '/pets/dogs/2', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( '/pets/fish/a', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( '/pets/fish/b', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( '/pets/fish_info.txt', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object( '/snakes', normalize_timestamp(0), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') #def list_objects_iter(self, limit, marker, prefix, delimiter, # path=None, format=None): listing = broker.list_objects_iter(100, None, None, '/pets/f', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b']) def test_double_check_trailing_delimiter(self): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object('a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/a/a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/a/b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b/a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b/b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('c', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a/0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('00', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/00', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/1', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/1/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0/1/0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1/0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) self.assertEquals( [row[0] for row in listing], ['0', '0/', '0/0', '0/00', '0/1', '0/1/', '0/1/0', '00', '1', '1/', '1/0', 'a', 'a/', 'a/0', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c']) listing = broker.list_objects_iter(25, None, None, '', '/') self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['0', '0/', '00', '1', '1/', 'a', 'a/', 'b', 'b/', 'c']) listing = broker.list_objects_iter(25, None, None, 'a/', '/') self.assertEquals(len(listing), 5) self.assertEquals( [row[0] for row in listing], ['a/', 'a/0', 'a/a', 'a/a/', 'a/b']) listing = broker.list_objects_iter(25, None, None, '0/', '/') self.assertEquals(len(listing), 5) self.assertEquals( [row[0] for row in listing], ['0/', '0/0', '0/00', '0/1', '0/1/']) listing = broker.list_objects_iter(25, None, None, '0/1/', '/') self.assertEquals(len(listing), 2) self.assertEquals( [row[0] for row in listing], ['0/1/', '0/1/0']) listing = broker.list_objects_iter(25, None, None, 'b/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) def test_double_check_trailing_delimiter_non_slash(self): # Test ContainerBroker.list_objects_iter for a # container that has an odd file with a trailing delimiter broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object('a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:a:a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:a:b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b:a', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b:b', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('c', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('a:0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('00', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:00', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:1', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:1:', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('0:1:0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1:', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('1:0', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') listing = broker.list_objects_iter(25, None, None, None, None) self.assertEquals(len(listing), 22) self.assertEquals( [row[0] for row in listing], ['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:', '1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b', 'b:a', 'b:b', 'c']) listing = broker.list_objects_iter(25, None, None, '', ':') self.assertEquals(len(listing), 10) self.assertEquals( [row[0] for row in listing], ['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c']) listing = broker.list_objects_iter(25, None, None, 'a:', ':') self.assertEquals(len(listing), 5) self.assertEquals( [row[0] for row in listing], ['a:', 'a:0', 'a:a', 'a:a:', 'a:b']) listing = broker.list_objects_iter(25, None, None, '0:', ':') self.assertEquals(len(listing), 5) self.assertEquals( [row[0] for row in listing], ['0:', '0:0', '0:00', '0:1', '0:1:']) listing = broker.list_objects_iter(25, None, None, '0:1:', ':') self.assertEquals(len(listing), 2) self.assertEquals( [row[0] for row in listing], ['0:1:', '0:1:0']) listing = broker.list_objects_iter(25, None, None, 'b:', ':') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['b:a', 'b:b']) def test_chexor(self): broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object('a', normalize_timestamp(1), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('b', normalize_timestamp(2), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest() hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest() hashc = ''.join( ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) broker.put_object('b', normalize_timestamp(3), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest() hashc = ''.join( ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb))) self.assertEquals(broker.get_info()['hash'], hashc) def test_newid(self): # test DatabaseBroker.newid broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) id = broker.get_info()['id'] broker.newid('someid') self.assertNotEquals(id, broker.get_info()['id']) def test_get_items_since(self): # test DatabaseBroker.get_items_since broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) broker.put_object('a', normalize_timestamp(1), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') max_row = broker.get_replication_info()['max_row'] broker.put_object('b', normalize_timestamp(2), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') items = broker.get_items_since(max_row, 1000) self.assertEquals(len(items), 1) self.assertEquals(items[0]['name'], 'b') def test_sync_merging(self): # exercise the DatabaseBroker sync functions a bit broker1 = ContainerBroker(':memory:', account='a', container='c') broker1.initialize(normalize_timestamp('1')) broker2 = ContainerBroker(':memory:', account='a', container='c') broker2.initialize(normalize_timestamp('1')) self.assertEquals(broker2.get_sync('12345'), -1) broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}]) broker2.merge_syncs(broker1.get_syncs()) self.assertEquals(broker2.get_sync('12345'), 3) def test_merge_items(self): broker1 = ContainerBroker(':memory:', account='a', container='c') broker1.initialize(normalize_timestamp('1')) broker2 = ContainerBroker(':memory:', account='a', container='c') broker2.initialize(normalize_timestamp('1')) broker1.put_object('a', normalize_timestamp(1), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker1.put_object('b', normalize_timestamp(2), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') id = broker1.get_info()['id'] broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 2) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) broker1.put_object('c', normalize_timestamp(3), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(len(items), 3) self.assertEquals(['a', 'b', 'c'], sorted([rec['name'] for rec in items])) def test_merge_items_overwrite(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') broker1.initialize(normalize_timestamp('1')) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') broker2.initialize(normalize_timestamp('1')) broker1.put_object('a', normalize_timestamp(2), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker1.put_object('b', normalize_timestamp(3), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) broker1.put_object('a', normalize_timestamp(4), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': self.assertEquals(rec['created_at'], normalize_timestamp(4)) if rec['name'] == 'b': self.assertEquals(rec['created_at'], normalize_timestamp(3)) def test_merge_items_post_overwrite_out_of_order(self): # test DatabaseBroker.merge_items broker1 = ContainerBroker(':memory:', account='a', container='c') broker1.initialize(normalize_timestamp('1')) id = broker1.get_info()['id'] broker2 = ContainerBroker(':memory:', account='a', container='c') broker2.initialize(normalize_timestamp('1')) broker1.put_object('a', normalize_timestamp(2), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker1.put_object('b', normalize_timestamp(3), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) broker1.put_object('a', normalize_timestamp(4), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': self.assertEquals(rec['created_at'], normalize_timestamp(4)) if rec['name'] == 'b': self.assertEquals(rec['created_at'], normalize_timestamp(3)) self.assertEquals(rec['content_type'], 'text/plain') items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': self.assertEquals(rec['created_at'], normalize_timestamp(4)) if rec['name'] == 'b': self.assertEquals(rec['created_at'], normalize_timestamp(3)) broker1.put_object('b', normalize_timestamp(5), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker2.merge_items(broker1.get_items_since( broker2.get_sync(id), 1000), id) items = broker2.get_items_since(-1, 1000) self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items])) for rec in items: if rec['name'] == 'a': self.assertEquals(rec['created_at'], normalize_timestamp(4)) if rec['name'] == 'b': self.assertEquals(rec['created_at'], normalize_timestamp(5)) self.assertEquals(rec['content_type'], 'text/plain') def premetadata_create_container_stat_table(self, conn, put_timestamp=None): """ Copied from ContainerBroker before the metadata column was added; used for testing with TestContainerBrokerBeforeMetadata. Create the container_stat table which is specifc to the container DB. :param conn: DB connection object :param put_timestamp: put timestamp """ if put_timestamp is None: put_timestamp = normalize_timestamp(0) conn.executescript(''' CREATE TABLE container_stat ( account TEXT, container TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', object_count INTEGER, bytes_used INTEGER, reported_put_timestamp TEXT DEFAULT '0', reported_delete_timestamp TEXT DEFAULT '0', reported_object_count INTEGER DEFAULT 0, reported_bytes_used INTEGER DEFAULT 0, hash TEXT default '00000000000000000000000000000000', id TEXT, status TEXT DEFAULT '', status_changed_at TEXT DEFAULT '0' ); INSERT INTO container_stat (object_count, bytes_used) VALUES (0, 0); ''') conn.execute(''' UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? ''', (self.account, self.container, normalize_timestamp(time()), str(uuid4()), put_timestamp)) class TestContainerBrokerBeforeMetadata(TestContainerBroker): """ Tests for ContainerBroker against databases created before the metadata column was added. """ def setUp(self): self._imported_create_container_stat_table = \ ContainerBroker.create_container_stat_table ContainerBroker.create_container_stat_table = \ premetadata_create_container_stat_table broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) exc = None with broker.get() as conn: try: conn.execute('SELECT metadata FROM container_stat') except BaseException as err: exc = err self.assert_('no such column: metadata' in str(exc)) def tearDown(self): ContainerBroker.create_container_stat_table = \ self._imported_create_container_stat_table broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('SELECT metadata FROM container_stat') def prexsync_create_container_stat_table(self, conn, put_timestamp=None): """ Copied from ContainerBroker before the x_container_sync_point[12] columns were added; used for testing with TestContainerBrokerBeforeXSync. Create the container_stat table which is specifc to the container DB. :param conn: DB connection object :param put_timestamp: put timestamp """ if put_timestamp is None: put_timestamp = normalize_timestamp(0) conn.executescript(""" CREATE TABLE container_stat ( account TEXT, container TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', object_count INTEGER, bytes_used INTEGER, reported_put_timestamp TEXT DEFAULT '0', reported_delete_timestamp TEXT DEFAULT '0', reported_object_count INTEGER DEFAULT 0, reported_bytes_used INTEGER DEFAULT 0, hash TEXT default '00000000000000000000000000000000', id TEXT, status TEXT DEFAULT '', status_changed_at TEXT DEFAULT '0', metadata TEXT DEFAULT '' ); INSERT INTO container_stat (object_count, bytes_used) VALUES (0, 0); """) conn.execute(''' UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? ''', (self.account, self.container, normalize_timestamp(time()), str(uuid4()), put_timestamp)) class TestContainerBrokerBeforeXSync(TestContainerBroker): """ Tests for ContainerBroker against databases created before the x_container_sync_point[12] columns were added. """ def setUp(self): self._imported_create_container_stat_table = \ ContainerBroker.create_container_stat_table ContainerBroker.create_container_stat_table = \ prexsync_create_container_stat_table broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) exc = None with broker.get() as conn: try: conn.execute('''SELECT x_container_sync_point1 FROM container_stat''') except BaseException as err: exc = err self.assert_('no such column: x_container_sync_point1' in str(exc)) def tearDown(self): ContainerBroker.create_container_stat_table = \ self._imported_create_container_stat_table broker = ContainerBroker(':memory:', account='a', container='c') broker.initialize(normalize_timestamp('1')) with broker.get() as conn: conn.execute('SELECT x_container_sync_point1 FROM container_stat') swift-1.13.1/test/unit/container/__init__.py0000664000175400017540000000000012323703611022104 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/proxy/0000775000175400017540000000000012323703665017215 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/proxy/test_server.py0000664000175400017540000102334412323703614022135 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import cPickle as pickle import logging import os import sys import unittest from contextlib import contextmanager, nested, closing from gzip import GzipFile from shutil import rmtree import gc import time from urllib import quote from hashlib import md5 from tempfile import mkdtemp import weakref import re import mock from eventlet import sleep, spawn, wsgi, listen import simplejson from test.unit import connect_tcp, readuntil2crlfs, FakeLogger, \ fake_http_connect, FakeRing, FakeMemcache, debug_logger from swift.proxy import server as proxy_server from swift.account import server as account_server from swift.container import server as container_server from swift.obj import server as object_server from swift.common import ring from swift.common.middleware import proxy_logging from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout from swift.common.constraints import MAX_META_NAME_LENGTH, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ ACCOUNT_LISTING_LIMIT, CONTAINER_LISTING_LIMIT, MAX_OBJECT_NAME_LENGTH from swift.common import utils from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_memcache_key, cors_validation import swift.proxy.controllers from swift.common.request_helpers import get_sys_meta_prefix from swift.common.swob import Request, Response, HTTPUnauthorized, \ HTTPException # mocks logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) STATIC_TIME = time.time() _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ _testdir = _orig_SysLogHandler = None def do_setup(the_object_server): utils.HASH_PATH_SUFFIX = 'endcap' global _testdir, _test_servers, _test_sockets, \ _orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_SysLogHandler = utils.SysLogHandler utils.SysLogHandler = mock.MagicMock() monkey_patch_mimetools() # Since we're starting up a lot here, we're going to test more than # just chunked puts; we're also going to test parts of # proxy_server.Application we couldn't get to easily otherwise. _testdir = \ os.path.join(mkdtemp(), 'tmp_test_proxy_server_chunked') mkdirs(_testdir) rmtree(_testdir) mkdirs(os.path.join(_testdir, 'sda1')) mkdirs(os.path.join(_testdir, 'sda1', 'tmp')) mkdirs(os.path.join(_testdir, 'sdb1')) mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) conf = {'devices': _testdir, 'swift_dir': _testdir, 'mount_check': 'false', 'allowed_headers': 'content-encoding, x-object-manifest, content-disposition, foo', 'allow_versions': 'True'} prolis = listen(('localhost', 0)) acc1lis = listen(('localhost', 0)) acc2lis = listen(('localhost', 0)) con1lis = listen(('localhost', 0)) con2lis = listen(('localhost', 0)) obj1lis = listen(('localhost', 0)) obj2lis = listen(('localhost', 0)) _test_sockets = \ (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) account_ring_path = os.path.join(_testdir, 'account.ring.gz') with closing(GzipFile(account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': acc1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': acc2lis.getsockname()[1]}], 30), f) container_ring_path = os.path.join(_testdir, 'container.ring.gz') with closing(GzipFile(container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': con1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': con2lis.getsockname()[1]}], 30), f) object_ring_path = os.path.join(_testdir, 'object.ring.gz') with closing(GzipFile(object_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': obj1lis.getsockname()[1]}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1', 'port': obj2lis.getsockname()[1]}], 30), f) prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone(), logger=debug_logger('proxy')) acc1srv = account_server.AccountController( conf, logger=debug_logger('acct1')) acc2srv = account_server.AccountController( conf, logger=debug_logger('acct2')) con1srv = container_server.ContainerController( conf, logger=debug_logger('cont1')) con2srv = container_server.ContainerController( conf, logger=debug_logger('cont2')) obj1srv = the_object_server.ObjectController( conf, logger=debug_logger('obj1')) obj2srv = the_object_server.ObjectController( conf, logger=debug_logger('obj2')) _test_servers = \ (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) nl = NullLogger() logging_prosv = proxy_logging.ProxyLoggingMiddleware(prosrv, conf, logger=prosrv.logger) prospa = spawn(wsgi.server, prolis, logging_prosv, nl) acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl) acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl) con1spa = spawn(wsgi.server, con1lis, con1srv, nl) con2spa = spawn(wsgi.server, con2lis, con2srv, nl) obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl) obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl) _test_coros = \ (prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa) # Create account ts = normalize_timestamp(time.time()) partition, nodes = prosrv.account_ring.get_nodes('a') for node in nodes: conn = swift.proxy.controllers.obj.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create container sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % ( exp, headers[:len(exp)]) def setup(): do_setup(object_server) def teardown(): for server in _test_coros: server.kill() rmtree(os.path.dirname(_testdir)) utils.SysLogHandler = _orig_SysLogHandler def sortHeaderNames(headerNames): """ Return the given string of header names sorted. headerName: a comma-delimited list of header names """ headers = [a.strip() for a in headerNames.split(',') if a.strip()] headers.sort() return ', '.join(headers) class FakeMemcacheReturnsNone(FakeMemcache): def get(self, key): # Returns None as the timestamp of the container; assumes we're only # using the FakeMemcache for container existence checks. return None @contextmanager def save_globals(): orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect', None) orig_account_info = getattr(swift.proxy.controllers.Controller, 'account_info', None) try: yield True finally: swift.proxy.controllers.Controller.account_info = orig_account_info swift.proxy.controllers.base.http_connect = orig_http_connect swift.proxy.controllers.obj.http_connect = orig_http_connect swift.proxy.controllers.account.http_connect = orig_http_connect swift.proxy.controllers.container.http_connect = orig_http_connect def set_http_connect(*args, **kwargs): new_connect = fake_http_connect(*args, **kwargs) swift.proxy.controllers.base.http_connect = new_connect swift.proxy.controllers.obj.http_connect = new_connect swift.proxy.controllers.account.http_connect = new_connect swift.proxy.controllers.container.http_connect = new_connect return new_connect # tests class TestController(unittest.TestCase): def setUp(self): self.account_ring = FakeRing() self.container_ring = FakeRing() self.memcache = FakeMemcache() app = proxy_server.Application(None, self.memcache, account_ring=self.account_ring, container_ring=self.container_ring, object_ring=FakeRing()) self.controller = swift.proxy.controllers.Controller(app) class FakeReq(object): def __init__(self): self.url = "/foo/bar" self.method = "METHOD" def as_referer(self): return self.method + ' ' + self.url self.account = 'some_account' self.container = 'some_container' self.request = FakeReq() self.read_acl = 'read_acl' self.write_acl = 'write_acl' def test_transfer_headers(self): src_headers = {'x-remove-base-meta-owner': 'x', 'x-base-meta-size': '151M', 'new-owner': 'Kun'} dst_headers = {'x-base-meta-owner': 'Gareth', 'x-base-meta-size': '150M'} self.controller.transfer_headers(src_headers, dst_headers) expected_headers = {'x-base-meta-owner': '', 'x-base-meta-size': '151M'} self.assertEquals(dst_headers, expected_headers) def check_account_info_return(self, partition, nodes, is_none=False): if is_none: p, n = None, None else: p, n = self.account_ring.get_nodes(self.account) self.assertEqual(p, partition) self.assertEqual(n, nodes) def test_account_info_container_count(self): with save_globals(): set_http_connect(200, count=123) partition, nodes, count = \ self.controller.account_info(self.account) self.assertEquals(count, 123) with save_globals(): set_http_connect(200, count='123') partition, nodes, count = \ self.controller.account_info(self.account) self.assertEquals(count, 123) with save_globals(): cache_key = get_account_memcache_key(self.account) account_info = {'status': 200, 'container_count': 1234} self.memcache.set(cache_key, account_info) partition, nodes, count = \ self.controller.account_info(self.account) self.assertEquals(count, 1234) with save_globals(): cache_key = get_account_memcache_key(self.account) account_info = {'status': 200, 'container_count': '1234'} self.memcache.set(cache_key, account_info) partition, nodes, count = \ self.controller.account_info(self.account) self.assertEquals(count, 1234) def test_make_requests(self): with save_globals(): set_http_connect(200) partition, nodes, count = \ self.controller.account_info(self.account, self.request) set_http_connect(201, raise_timeout_exc=True) self.controller._make_request( nodes, partition, 'POST', '/', '', '', self.controller.app.logger.thread_locals) # tests if 200 is cached and used def test_account_info_200(self): with save_globals(): set_http_connect(200) partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes) self.assertEquals(count, 12345) # Test the internal representation in memcache # 'container_count' changed from int to str cache_key = get_account_memcache_key(self.account) container_info = {'status': 200, 'container_count': '12345', 'total_object_count': None, 'bytes': None, 'meta': {}, 'sysmeta': {}} self.assertEquals(container_info, self.memcache.get(cache_key)) set_http_connect() partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes) self.assertEquals(count, 12345) # tests if 404 is cached and used def test_account_info_404(self): with save_globals(): set_http_connect(404, 404, 404) partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) # Test the internal representation in memcache # 'container_count' changed from 0 to None cache_key = get_account_memcache_key(self.account) account_info = {'status': 404, 'container_count': None, # internally keep None 'total_object_count': None, 'bytes': None, 'meta': {}, 'sysmeta': {}} self.assertEquals(account_info, self.memcache.get(cache_key)) set_http_connect() partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) # tests if some http status codes are not cached def test_account_info_no_cache(self): def test(*status_list): set_http_connect(*status_list) partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.assertEqual(len(self.memcache.keys()), 0) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) with save_globals(): # We cache if we have two 404 responses - fail if only one test(503, 503, 404) test(504, 404, 503) test(404, 507, 503) test(503, 503, 503) def test_account_info_no_account(self): with save_globals(): self.memcache.store = {} set_http_connect(404, 404, 404) partition, nodes, count = \ self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(count, None) def check_container_info_return(self, ret, is_none=False): if is_none: partition, nodes, read_acl, write_acl = None, None, None, None else: partition, nodes = self.container_ring.get_nodes(self.account, self.container) read_acl, write_acl = self.read_acl, self.write_acl self.assertEqual(partition, ret['partition']) self.assertEqual(nodes, ret['nodes']) self.assertEqual(read_acl, ret['read_acl']) self.assertEqual(write_acl, ret['write_acl']) def test_container_info_invalid_account(self): def account_info(self, account, request, autocreate=False): return None, None with save_globals(): swift.proxy.controllers.Controller.account_info = account_info ret = self.controller.container_info(self.account, self.container, self.request) self.check_container_info_return(ret, True) # tests if 200 is cached and used def test_container_info_200(self): with save_globals(): headers = {'x-container-read': self.read_acl, 'x-container-write': self.write_acl} set_http_connect(200, # account_info is found 200, headers=headers) # container_info is found ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret) cache_key = get_container_memcache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEquals(200, cache_value.get('status')) set_http_connect() ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret) # tests if 404 is cached and used def test_container_info_404(self): def account_info(self, account, request): return True, True, 0 with save_globals(): set_http_connect(503, 204, # account_info found 504, 404, 404) # container_info 'NotFound' ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) cache_key = get_container_memcache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEquals(404, cache_value.get('status')) set_http_connect() ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) set_http_connect(503, 404, 404) # account_info 'NotFound' ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) cache_key = get_container_memcache_key(self.account, self.container) cache_value = self.memcache.get(cache_key) self.assertTrue(isinstance(cache_value, dict)) self.assertEquals(404, cache_value.get('status')) set_http_connect() ret = self.controller.container_info( self.account, self.container, self.request) self.check_container_info_return(ret, True) # tests if some http status codes are not cached def test_container_info_no_cache(self): def test(*status_list): set_http_connect(*status_list) ret = self.controller.container_info( self.account, self.container, self.request) self.assertEqual(len(self.memcache.keys()), 0) self.check_container_info_return(ret, True) with save_globals(): # We cache if we have two 404 responses - fail if only one test(503, 503, 404) test(504, 404, 503) test(404, 507, 503) test(503, 503, 503) class TestProxyServer(unittest.TestCase): def test_unhandled_exception(self): class MyApp(proxy_server.Application): def get_controller(self, path): raise Exception('this shouldnt be caught') app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) app.update_request(req) resp = app.handle_request(req) self.assertEquals(resp.status_int, 500) def test_internal_method_request(self): baseapp = proxy_server.Application({}, FakeMemcache(), container_ring=FakeRing(), object_ring=FakeRing(), account_ring=FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a', environ={'REQUEST_METHOD': '__init__'})) self.assertEquals(resp.status, '405 Method Not Allowed') def test_inexistent_method_request(self): baseapp = proxy_server.Application({}, FakeMemcache(), container_ring=FakeRing(), account_ring=FakeRing(), object_ring=FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a', environ={'REQUEST_METHOD': '!invalid'})) self.assertEquals(resp.status, '405 Method Not Allowed') def test_calls_authorize_allow(self): called = [False] def authorize(req): called[0] = True with save_globals(): set_http_connect(200) app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) req = Request.blank('/v1/a') req.environ['swift.authorize'] = authorize app.update_request(req) app.handle_request(req) self.assert_(called[0]) def test_calls_authorize_deny(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) req = Request.blank('/v1/a') req.environ['swift.authorize'] = authorize app.update_request(req) app.handle_request(req) self.assert_(called[0]) def test_negative_content_length(self): swift_dir = mkdtemp() try: baseapp = proxy_server.Application({'swift_dir': swift_dir}, FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), FakeRing()) resp = baseapp.handle_request( Request.blank('/', environ={'CONTENT_LENGTH': '-1'})) self.assertEquals(resp.status, '400 Bad Request') self.assertEquals(resp.body, 'Invalid Content-Length') resp = baseapp.handle_request( Request.blank('/', environ={'CONTENT_LENGTH': '-123'})) self.assertEquals(resp.status, '400 Bad Request') self.assertEquals(resp.body, 'Invalid Content-Length') finally: rmtree(swift_dir, ignore_errors=True) def test_denied_host_header(self): swift_dir = mkdtemp() try: baseapp = proxy_server.Application({'swift_dir': swift_dir, 'deny_host_headers': 'invalid_host.com'}, FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), FakeRing()) resp = baseapp.handle_request( Request.blank('/v1/a/c/o', environ={'HTTP_HOST': 'invalid_host.com'})) self.assertEquals(resp.status, '403 Forbidden') finally: rmtree(swift_dir, ignore_errors=True) def test_node_timing(self): baseapp = proxy_server.Application({'sorting_method': 'timing'}, FakeMemcache(), container_ring=FakeRing(), object_ring=FakeRing(), account_ring=FakeRing()) self.assertEquals(baseapp.node_timings, {}) req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) baseapp.update_request(req) resp = baseapp.handle_request(req) self.assertEquals(resp.status_int, 503) # couldn't connect to anything exp_timings = {} self.assertEquals(baseapp.node_timings, exp_timings) times = [time.time()] exp_timings = {'127.0.0.1': (0.1, times[0] + baseapp.timing_expiry)} with mock.patch('swift.proxy.server.time', lambda: times.pop(0)): baseapp.set_node_timing({'ip': '127.0.0.1'}, 0.1) self.assertEquals(baseapp.node_timings, exp_timings) nodes = [{'ip': '127.0.0.1'}, {'ip': '127.0.0.2'}, {'ip': '127.0.0.3'}] with mock.patch('swift.proxy.server.shuffle', lambda l: l): res = baseapp.sort_nodes(nodes) exp_sorting = [{'ip': '127.0.0.2'}, {'ip': '127.0.0.3'}, {'ip': '127.0.0.1'}] self.assertEquals(res, exp_sorting) def test_node_affinity(self): baseapp = proxy_server.Application({'sorting_method': 'affinity', 'read_affinity': 'r1=1'}, FakeMemcache(), container_ring=FakeRing(), object_ring=FakeRing(), account_ring=FakeRing()) nodes = [{'region': 2, 'zone': 1, 'ip': '127.0.0.1'}, {'region': 1, 'zone': 2, 'ip': '127.0.0.2'}] with mock.patch('swift.proxy.server.shuffle', lambda x: x): app_sorted = baseapp.sort_nodes(nodes) exp_sorted = [{'region': 1, 'zone': 2, 'ip': '127.0.0.2'}, {'region': 2, 'zone': 1, 'ip': '127.0.0.1'}] self.assertEquals(exp_sorted, app_sorted) def test_info_defaults(self): app = proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) self.assertTrue(app.expose_info) self.assertTrue(isinstance(app.disallowed_sections, list)) self.assertEqual(0, len(app.disallowed_sections)) self.assertTrue(app.admin_key is None) def test_get_info_controller(self): path = '/info' app = proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) controller, path_parts = app.get_controller(path) self.assertTrue('version' in path_parts) self.assertTrue(path_parts['version'] is None) self.assertTrue('disallowed_sections' in path_parts) self.assertTrue('expose_info' in path_parts) self.assertTrue('admin_key' in path_parts) self.assertEqual(controller.__name__, 'InfoController') class TestObjectController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), logger=debug_logger('proxy-ut'), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) def tearDown(self): self.app.account_ring.set_replicas(3) self.app.container_ring.set_replicas(3) self.app.object_ring.set_replicas(3) def assert_status_map(self, method, statuses, expected, raise_exc=False): with save_globals(): kwargs = {} if raise_exc: kwargs['raise_exc'] = raise_exc set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) # repeat test set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) def test_GET_newest_large_file(self): prolis = _test_sockets[0] prosrv = _test_servers[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() obj = 'a' * (1024 * 1024) path = '/v1/a/c/o.large' fd.write('PUT %s HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' '\r\n%s' % (path, str(len(obj)), obj)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) req = Request.blank(path, environ={'REQUEST_METHOD': 'GET'}, headers={'Content-Type': 'application/octet-stream', 'X-Newest': 'true'}) res = req.get_response(prosrv) self.assertEqual(res.status_int, 200) self.assertEqual(res.body, obj) def test_PUT_expect_header_zero_content_length(self): test_errors = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.jpg': if 'expect' in headers or 'Expect' in headers: test_errors.append('Expect was in headers for object ' 'server!') with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') # The (201, -4) tuples in there have the effect of letting the # initial connect succeed, after which getexpect() gets called and # then the -4 makes the response of that actually be 201 instead of # 100. Perfectly straightforward. set_http_connect(200, 200, (201, -4), (201, -4), (201, -4), give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 '), res.status) def test_PUT_expect_header_nonzero_content_length(self): test_errors = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.jpg': if 'Expect' not in headers: test_errors.append('Expect was not in headers for ' 'non-zero byte PUT!') with save_globals(): controller = \ proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') set_http_connect(200, 200, 201, 201, 201, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 1 req.body = 'a' self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) self.assertEqual(test_errors, []) self.assertTrue(res.status.startswith('201 ')) def test_PUT_respects_write_affinity(self): written_to = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.jpg': written_to.append((ipaddr, port, device)) with save_globals(): def is_r0(node): return node['region'] == 0 self.app.object_ring.max_more_nodes = 100 self.app.write_affinity_is_local_fn = is_r0 self.app.write_affinity_node_count = lambda r: 3 controller = \ proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') set_http_connect(200, 200, 201, 201, 201, give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 1 req.body = 'a' self.app.memcache.store = {} res = controller.PUT(req) self.assertTrue(res.status.startswith('201 ')) self.assertEqual(3, len(written_to)) for ip, port, device in written_to: # this is kind of a hokey test, but in FakeRing, the port is even # when the region is 0, and odd when the region is 1, so this test # asserts that we only wrote to nodes in region 0. self.assertEqual(0, port % 2) def test_PUT_respects_write_affinity_with_507s(self): written_to = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.jpg': written_to.append((ipaddr, port, device)) with save_globals(): def is_r0(node): return node['region'] == 0 self.app.object_ring.max_more_nodes = 100 self.app.write_affinity_is_local_fn = is_r0 self.app.write_affinity_node_count = lambda r: 3 controller = \ proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') self.app.error_limit( self.app.object_ring.get_part_nodes(1)[0], 'test') set_http_connect(200, 200, # account, container 201, 201, 201, # 3 working backends give_connect=test_connect) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 1 req.body = 'a' self.app.memcache.store = {} res = controller.PUT(req) self.assertTrue(res.status.startswith('201 ')) self.assertEqual(3, len(written_to)) # this is kind of a hokey test, but in FakeRing, the port is even when # the region is 0, and odd when the region is 1, so this test asserts # that we wrote to 2 nodes in region 0, then went to 1 non-r0 node. self.assertEqual(0, written_to[0][1] % 2) # it's (ip, port, device) self.assertEqual(0, written_to[1][1] % 2) self.assertNotEqual(0, written_to[2][1] % 2) def test_PUT_message_length_using_content_length(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() obj = 'j' * 20 fd.write('PUT /v1/a/c/o.content-length HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' '\r\n%s' % (str(len(obj)), obj)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) def test_PUT_message_length_using_transfer_encoding(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.chunked HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Type: application/octet-stream\r\n' 'Transfer-Encoding: chunked\r\n\r\n' '2\r\n' 'oh\r\n' '4\r\n' ' say\r\n' '4\r\n' ' can\r\n' '4\r\n' ' you\r\n' '4\r\n' ' see\r\n' '3\r\n' ' by\r\n' '4\r\n' ' the\r\n' '8\r\n' ' dawns\'\n\r\n' '0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) def test_PUT_message_length_using_both(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.chunked HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Type: application/octet-stream\r\n' 'Content-Length: 33\r\n' 'Transfer-Encoding: chunked\r\n\r\n' '2\r\n' 'oh\r\n' '4\r\n' ' say\r\n' '4\r\n' ' can\r\n' '4\r\n' ' you\r\n' '4\r\n' ' see\r\n' '3\r\n' ' by\r\n' '4\r\n' ' the\r\n' '8\r\n' ' dawns\'\n\r\n' '0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) def test_PUT_bad_message_length(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.chunked HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Type: application/octet-stream\r\n' 'Content-Length: 33\r\n' 'Transfer-Encoding: gzip\r\n\r\n' '2\r\n' 'oh\r\n' '4\r\n' ' say\r\n' '4\r\n' ' can\r\n' '4\r\n' ' you\r\n' '4\r\n' ' see\r\n' '3\r\n' ' by\r\n' '4\r\n' ' the\r\n' '8\r\n' ' dawns\'\n\r\n' '0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 400' self.assertEqual(headers[:len(exp)], exp) def test_PUT_message_length_unsup_xfr_encoding(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.chunked HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Type: application/octet-stream\r\n' 'Content-Length: 33\r\n' 'Transfer-Encoding: gzip,chunked\r\n\r\n' '2\r\n' 'oh\r\n' '4\r\n' ' say\r\n' '4\r\n' ' can\r\n' '4\r\n' ' you\r\n' '4\r\n' ' see\r\n' '3\r\n' ' by\r\n' '4\r\n' ' the\r\n' '8\r\n' ' dawns\'\n\r\n' '0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 501' self.assertEqual(headers[:len(exp)], exp) def test_PUT_message_length_too_large(self): swift.proxy.controllers.obj.MAX_FILE_SIZE = 10 try: prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.chunked HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Type: application/octet-stream\r\n' 'Content-Length: 33\r\n\r\n' 'oh say can you see by the dawns\'\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 413' self.assertEqual(headers[:len(exp)], exp) finally: swift.proxy.controllers.obj.MAX_FILE_SIZE = MAX_FILE_SIZE def test_PUT_last_modified(self): prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/o.last_modified HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' lm_hdr = 'Last-Modified: ' self.assertEqual(headers[:len(exp)], exp) last_modified_put = [line for line in headers.split('\r\n') if lm_hdr in line][0][len(lm_hdr):] sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('HEAD /v1/a/c/o.last_modified HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEqual(headers[:len(exp)], exp) last_modified_head = [line for line in headers.split('\r\n') if lm_hdr in line][0][len(lm_hdr):] self.assertEqual(last_modified_put, last_modified_head) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'If-Modified-Since: %s\r\n' 'X-Storage-Token: t\r\n\r\n' % last_modified_put) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 304' self.assertEqual(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/c/o.last_modified HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'If-Unmodified-Since: %s\r\n' 'X-Storage-Token: t\r\n\r\n' % last_modified_put) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEqual(headers[:len(exp)], exp) def test_expirer_DELETE_on_versioned_object(self): test_errors = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if method == 'DELETE': if 'x-if-delete-at' in headers or 'X-If-Delete-At' in headers: test_errors.append('X-If-Delete-At in headers') body = simplejson.dumps( [{"name": "001o/1", "hash": "x", "bytes": 0, "content_type": "text/plain", "last_modified": "1970-01-01T00:00:01.000000"}]) body_iter = ('', '', body, '', '', '', '', '', '', '', '', '', '', '') with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') # HEAD HEAD GET GET HEAD GET GET GET PUT PUT # PUT DEL DEL DEL set_http_connect(200, 200, 200, 200, 200, 200, 200, 200, 201, 201, 201, 200, 200, 200, give_connect=test_connect, body_iter=body_iter, headers={'x-versions-location': 'foo'}) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', headers={'X-If-Delete-At': 1}, environ={'REQUEST_METHOD': 'DELETE'}) self.app.update_request(req) controller.DELETE(req) self.assertEquals(test_errors, []) def test_PUT_auto_content_type(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') def test_content_type(filename, expected): # The three responses here are for account_info() (HEAD to # account server), container_info() (HEAD to container server) # and three calls to _connect_put_node() (PUT to three object # servers) set_http_connect(201, 201, 201, 201, 201, give_content_type=lambda content_type: self.assertEquals(content_type, expected.next())) # We need into include a transfer-encoding to get past # constraints.check_object_creation() req = Request.blank('/v1/a/c/%s' % filename, {}, headers={'transfer-encoding': 'chunked'}) self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) # If we don't check the response here we could miss problems # in PUT() self.assertEquals(res.status_int, 201) test_content_type('test.jpg', iter(['', '', 'image/jpeg', 'image/jpeg', 'image/jpeg'])) test_content_type('test.html', iter(['', '', 'text/html', 'text/html', 'text/html'])) test_content_type('test.css', iter(['', '', 'text/css', 'text/css', 'text/css'])) def test_custom_mime_types_files(self): swift_dir = mkdtemp() try: with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp: fp.write('foo/bar foo\n') proxy_server.Application({'swift_dir': swift_dir}, FakeMemcache(), FakeLogger(), FakeRing(), FakeRing(), FakeRing()) self.assertEquals(proxy_server.mimetypes.guess_type('blah.foo')[0], 'foo/bar') self.assertEquals(proxy_server.mimetypes.guess_type('blah.jpg')[0], 'image/jpeg') finally: rmtree(swift_dir, ignore_errors=True) def test_PUT(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 201, 201, 201), 201) test_status_map((200, 200, 201, 201, 500), 201) test_status_map((200, 200, 204, 404, 404), 404) test_status_map((200, 200, 204, 500, 404), 503) def test_PUT_connect_exceptions(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 201, 201, -1), 201) test_status_map((200, 200, 201, 201, -2), 201) # expect timeout test_status_map((200, 200, 201, 201, -3), 201) # error limited test_status_map((200, 200, 201, -1, -1), 503) test_status_map((200, 200, 503, 503, -1), 503) def test_PUT_send_exceptions(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} set_http_connect(*statuses) req = Request.blank('/v1/a/c/o.jpg', environ={'REQUEST_METHOD': 'PUT'}, body='some data') self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 201, -1, 201), 201) test_status_map((200, 200, 201, -1, -1), 503) test_status_map((200, 200, 503, 503, -1), 503) def test_PUT_max_size(self): with save_globals(): set_http_connect(201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ 'Content-Length': str(MAX_FILE_SIZE + 1), 'Content-Type': 'foo/bar'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 413) def test_PUT_bad_content_type(self): with save_globals(): set_http_connect(201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {}, headers={ 'Content-Length': 0, 'Content-Type': 'foo/bar;swift_hey=45'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 400) def test_PUT_getresponse_exceptions(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') def test_status_map(statuses, expected): self.app.memcache.store = {} set_http_connect(*statuses) req = Request.blank('/v1/a/c/o.jpg', {}) req.content_length = 0 self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(str(expected))], str(expected)) test_status_map((200, 200, 201, 201, -1), 201) test_status_map((200, 200, 201, -1, -1), 503) test_status_map((200, 200, 503, 503, -1), 503) def test_POST(self): with save_globals(): self.app.object_post_as_copy = False def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, method='POST', headers={'Content-Type': 'foo/bar'}) self.app.update_request(req) res = req.get_response(self.app) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 202, 202, 202), 202) test_status_map((200, 200, 202, 202, 500), 202) test_status_map((200, 200, 202, 500, 500), 503) test_status_map((200, 200, 202, 404, 500), 503) test_status_map((200, 200, 202, 404, 404), 404) test_status_map((200, 200, 404, 500, 500), 503) test_status_map((200, 200, 404, 404, 404), 404) def test_POST_as_copy(self): with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar'}) self.app.update_request(req) res = req.get_response(self.app) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202) test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503) test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503) test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404) test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503) test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404) def test_DELETE(self): with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) test_status_map((200, 200, 204, 204, 204), 204) test_status_map((200, 200, 204, 204, 500), 204) test_status_map((200, 200, 204, 404, 404), 404) test_status_map((200, 200, 204, 500, 404), 503) test_status_map((200, 200, 404, 404, 404), 404) test_status_map((200, 200, 404, 404, 500), 404) def test_HEAD(self): with save_globals(): def test_status_map(statuses, expected): set_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) if expected < 400: self.assert_('x-works' in res.headers) self.assertEquals(res.headers['x-works'], 'yes') self.assert_('accept-ranges' in res.headers) self.assertEquals(res.headers['accept-ranges'], 'bytes') test_status_map((200, 200, 200, 404, 404), 200) test_status_map((200, 200, 200, 500, 404), 200) test_status_map((200, 200, 304, 500, 404), 304) test_status_map((200, 200, 404, 404, 404), 404) test_status_map((200, 200, 404, 404, 500), 404) test_status_map((200, 200, 500, 500, 500), 503) def test_HEAD_newest(self): with save_globals(): def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}, headers={'x-newest': 'true'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), expected_timestamp) # acct cont obj obj obj test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1') def test_GET_newest(self): with save_globals(): def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'}, headers={'x-newest': 'true'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), expected_timestamp) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, None), None) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, None, '1'), '1') with save_globals(): def test_status_map(statuses, expected, timestamps, expected_timestamp): set_http_connect(*statuses, timestamps=timestamps) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status[:len(str(expected))], str(expected)) self.assertEquals(res.headers.get('last-modified'), expected_timestamp) test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '2', '3'), '1') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '2'), '1') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1', '3', '1'), '1') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3', '3', '1'), '3') test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None, '1', '2'), None) def test_POST_meta_val_len(self): with save_globals(): limit = MAX_META_VALUE_LENGTH self.app.object_post_as_copy = False proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x' * limit}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x' * (limit + 1)}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_as_copy_meta_val_len(self): with save_globals(): limit = MAX_META_VALUE_LENGTH set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x' * limit}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x' * (limit + 1)}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_key_len(self): with save_globals(): limit = MAX_META_NAME_LENGTH self.app.object_post_as_copy = False set_http_connect(200, 200, 202, 202, 202) # acct cont obj obj obj req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', ('X-Object-Meta-' + 'x' * limit): 'x'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_as_copy_meta_key_len(self): with save_globals(): limit = MAX_META_NAME_LENGTH set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) # acct cont objc objc objc obj obj obj req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', ('X-Object-Meta-' + 'x' * limit): 'x'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 202) set_http_connect(202, 202, 202) req = Request.blank( '/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'foo/bar', ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'}) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_count(self): with save_globals(): limit = MAX_META_COUNT headers = dict( (('X-Object-Meta-' + str(i), 'a') for i in xrange(limit + 1))) headers.update({'Content-Type': 'foo/bar'}) set_http_connect(202, 202, 202) req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers=headers) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_POST_meta_size(self): with save_globals(): limit = MAX_META_OVERALL_SIZE count = limit / 256 # enough to cause the limit to be reached headers = dict( (('X-Object-Meta-' + str(i), 'a' * 256) for i in xrange(count + 1))) headers.update({'Content-Type': 'foo/bar'}) set_http_connect(202, 202, 202) req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'}, headers=headers) self.app.update_request(req) res = req.get_response(self.app) self.assertEquals(res.status_int, 400) def test_PUT_not_autodetect_content_type(self): with save_globals(): headers = {'Content-Type': 'something/right', 'Content-Length': 0} it_worked = [] def verify_content_type(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.html': it_worked.append( headers['Content-Type'].startswith('something/right')) set_http_connect(204, 204, 201, 201, 201, give_connect=verify_content_type) req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) req.get_response(self.app) self.assertNotEquals(it_worked, []) self.assertTrue(all(it_worked)) def test_PUT_autodetect_content_type(self): with save_globals(): headers = {'Content-Type': 'something/wrong', 'Content-Length': 0, 'X-Detect-Content-Type': 'True'} it_worked = [] def verify_content_type(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c/o.html': it_worked.append( headers['Content-Type'].startswith('text/html')) set_http_connect(204, 204, 201, 201, 201, give_connect=verify_content_type) req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) req.get_response(self.app) self.assertNotEquals(it_worked, []) self.assertTrue(all(it_worked)) def test_client_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') for dev in self.app.container_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.object_ring.get_nodes('account') for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 class SlowBody(object): def __init__(self): self.sent = 0 def read(self, size=-1): if self.sent < 4: sleep(0.1) self.sent += 1 return ' ' return '' req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) self.app.update_request(req) set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.client_timeout = 0.1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) self.app.update_request(req) set_http_connect(201, 201, 201) # obj obj obj resp = req.get_response(self.app) self.assertEquals(resp.status_int, 408) def test_client_disconnect(self): with save_globals(): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') for dev in self.app.container_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.object_ring.get_nodes('account') for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 class SlowBody(object): def __init__(self): self.sent = 0 def read(self, size=-1): raise Exception('Disconnected') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) self.app.update_request(req) set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj resp = req.get_response(self.app) self.assertEquals(resp.status_int, 499) def test_node_read_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') for dev in self.app.container_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.object_ring.get_nodes('account') for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) set_http_connect(200, 200, 200, slow=0.1) req.sent_size = 0 resp = req.get_response(self.app) got_exc = False try: resp.body except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=1.0) resp = req.get_response(self.app) got_exc = False try: resp.body except ChunkReadTimeout: got_exc = True self.assert_(got_exc) def test_node_read_timeout_retry(self): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) self.app.recoverable_node_timeout = 0.1 set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: self.assertEquals('', resp.body) except ChunkReadTimeout: got_exc = True self.assert_(got_exc) set_http_connect(200, 200, 200, body='lalala', slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: self.assertEquals(resp.body, 'lalala') except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) set_http_connect(200, 200, 200, body='lalala', slow=[1.0, 1.0], etags=['a', 'a', 'a']) resp = req.get_response(self.app) got_exc = False try: self.assertEquals(resp.body, 'lalala') except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) set_http_connect(200, 200, 200, body='lalala', slow=[1.0, 1.0], etags=['a', 'b', 'a']) resp = req.get_response(self.app) got_exc = False try: self.assertEquals(resp.body, 'lalala') except ChunkReadTimeout: got_exc = True self.assert_(not got_exc) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) set_http_connect(200, 200, 200, body='lalala', slow=[1.0, 1.0], etags=['a', 'b', 'b']) resp = req.get_response(self.app) got_exc = False try: resp.body except ChunkReadTimeout: got_exc = True self.assert_(got_exc) def test_node_write_timeout(self): with save_globals(): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.container_ring.get_nodes('account') for dev in self.app.container_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 self.app.object_ring.get_nodes('account') for dev in self.app.object_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) set_http_connect(200, 200, 201, 201, 201, slow=0.1) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 201) self.app.node_timeout = 0.1 req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') self.app.update_request(req) set_http_connect(201, 201, 201, slow=1.0) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 503) def test_iter_nodes(self): with save_globals(): try: self.app.object_ring.max_more_nodes = 2 partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] for node in self.app.iter_nodes(self.app.object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.app.object_ring.max_more_nodes = 20 self.app.request_node_count = lambda r: 20 partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] for node in self.app.iter_nodes(self.app.object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 9) self.app.log_handoffs = True self.app.logger = FakeLogger() self.app.object_ring.max_more_nodes = 2 partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] for node in self.app.iter_nodes(self.app.object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.assertEquals( self.app.logger.log_dict['warning'], [(('Handoff requested (1)',), {}), (('Handoff requested (2)',), {})]) self.app.log_handoffs = False self.app.logger = FakeLogger() self.app.object_ring.max_more_nodes = 2 partition, nodes = self.app.object_ring.get_nodes('account', 'container', 'object') collected_nodes = [] for node in self.app.iter_nodes(self.app.object_ring, partition): collected_nodes.append(node) self.assertEquals(len(collected_nodes), 5) self.assertEquals(self.app.logger.log_dict['warning'], []) finally: self.app.object_ring.max_more_nodes = 0 def test_iter_nodes_calls_sort_nodes(self): with mock.patch.object(self.app, 'sort_nodes') as sort_nodes: for node in self.app.iter_nodes(self.app.object_ring, 0): pass sort_nodes.assert_called_once_with( self.app.object_ring.get_part_nodes(0)) def test_iter_nodes_skips_error_limited(self): with mock.patch.object(self.app, 'sort_nodes', lambda n: n): first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) self.assertTrue(first_nodes[0] in second_nodes) self.app.error_limit(first_nodes[0], 'test') second_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) self.assertTrue(first_nodes[0] not in second_nodes) def test_iter_nodes_gives_extra_if_error_limited_inline(self): with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 6), mock.patch.object(self.app.object_ring, 'max_more_nodes', 99)): first_nodes = list(self.app.iter_nodes(self.app.object_ring, 0)) second_nodes = [] for node in self.app.iter_nodes(self.app.object_ring, 0): if not second_nodes: self.app.error_limit(node, 'test') second_nodes.append(node) self.assertEquals(len(first_nodes), 6) self.assertEquals(len(second_nodes), 7) def test_iter_nodes_with_custom_node_iter(self): node_list = [dict(id=n) for n in xrange(10)] with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 3)): got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, node_iter=iter(node_list))) self.assertEqual(node_list[:3], got_nodes) with nested( mock.patch.object(self.app, 'sort_nodes', lambda n: n), mock.patch.object(self.app, 'request_node_count', lambda r: 1000000)): got_nodes = list(self.app.iter_nodes(self.app.object_ring, 0, node_iter=iter(node_list))) self.assertEqual(node_list, got_nodes) def test_best_response_sets_headers(self): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object', headers=[{'X-Test': '1'}, {'X-Test': '2'}, {'X-Test': '3'}]) self.assertEquals(resp.headers['X-Test'], '1') def test_best_response_sets_etag(self): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object') self.assertEquals(resp.etag, None) resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3, 'Object', etag='68b329da9893e34099c7d8ad5cb9c940' ) self.assertEquals(resp.etag, '68b329da9893e34099c7d8ad5cb9c940') def test_proxy_passes_content_type(self): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) set_http_connect(200, 200, 200) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_type, 'x-application/test') set_http_connect(200, 200, 200) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 0) set_http_connect(200, 200, 200, slow=True) resp = req.get_response(self.app) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 4) def test_proxy_passes_content_length_on_head(self): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 0) set_http_connect(200, 200, 200, slow=True) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 4) def test_error_limiting(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') controller.app.sort_nodes = lambda l: l self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200), 200) self.assertEquals(controller.app.object_ring.devs[0]['errors'], 2) self.assert_('last_error' in controller.app.object_ring.devs[0]) for _junk in xrange(self.app.error_suppression_limit): self.assert_status_map(controller.HEAD, (200, 200, 503, 503, 503), 503) self.assertEquals(controller.app.object_ring.devs[0]['errors'], self.app.error_suppression_limit + 1) self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 503) self.assert_('last_error' in controller.app.object_ring.devs[0]) self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201, 201), 503) self.assert_status_map(controller.POST, (200, 200, 200, 200, 200, 200, 202, 202, 202), 503) self.assert_status_map(controller.DELETE, (200, 200, 200, 204, 204, 204), 503) self.app.error_suppression_interval = -300 self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200), 200) self.assertRaises(BaseException, self.assert_status_map, controller.DELETE, (200, 200, 200, 204, 204, 204), 503, raise_exc=True) def test_acc_or_con_missing_returns_404(self): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() for dev in self.app.account_ring.devs.values(): del dev['errors'] del dev['last_error'] for dev in self.app.container_ring.devs.values(): del dev['errors'] del dev['last_error'] controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) self.app.update_request(req) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 200) set_http_connect(404, 404, 404) # acct acct acct # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, 404) # acct acct acct # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 503, 404) # acct acct acct # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 503, 503) # acct acct acct # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 200, 204, 204, 204) # acct cont obj obj obj # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 204) set_http_connect(200, 404, 404, 404) # acct cont cont cont # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 503, 503, 503) # acct cont cont cont # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) for dev in self.app.account_ring.devs.values(): dev['errors'] = self.app.error_suppression_limit + 1 dev['last_error'] = time.time() set_http_connect(200) # acct [isn't actually called since everything # is error limited] # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) for dev in self.app.account_ring.devs.values(): dev['errors'] = 0 for dev in self.app.container_ring.devs.values(): dev['errors'] = self.app.error_suppression_limit + 1 dev['last_error'] = time.time() set_http_connect(200, 200) # acct cont [isn't actually called since # everything is error limited] # make sure to use a fresh request without cached env req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 404) def test_PUT_POST_requires_container_exist(self): with save_globals(): self.app.object_post_as_copy = False self.app.memcache = FakeMemcacheReturnsNone() controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 404, 404, 404, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}) self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 404) def test_PUT_POST_as_copy_requires_container_exist(self): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) set_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, 200) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}) self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 404) def test_bad_metadata(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} for x in xrange(MAX_META_COUNT): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} for x in xrange(MAX_META_COUNT + 1): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {'Content-Length': '0'} header_value = 'a' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - \ MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Object-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Object-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Object-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @contextmanager def controller_context(self, req, *args, **kwargs): _v, account, container, obj = utils.split_path(req.path, 4, 4, True) controller = proxy_server.ObjectController(self.app, account, container, obj) self.app.update_request(req) self.app.memcache.store = {} with save_globals(): new_connect = set_http_connect(*args, **kwargs) yield controller unused_status_list = [] while True: try: unused_status_list.append(new_connect.code_iter.next()) except StopIteration: break if unused_status_list: raise self.fail('UN-USED STATUS CODES: %r' % unused_status_list) def test_basic_put_with_x_copy_from(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c/o'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') def test_basic_put_with_x_copy_from_across_container(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c2/o'}) status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) # acct cont conc objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c2/o') def test_copy_non_zero_content_length(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5', 'X-Copy-From': 'c/o'}) status_list = (200, 200) # acct cont with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) def test_copy_with_slashes_in_x_copy_from(self): # extra source path parsing req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c/o/o2'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') def test_copy_with_spaces_in_x_copy_from(self): # space in soure path req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c/o%20o2'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o%20o2') def test_copy_with_leading_slash_in_x_copy_from(self): # repeat tests with leading / req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') def test_copy_with_leading_slash_and_slashes_in_x_copy_from(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o/o2'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') def test_copy_with_no_object_in_x_copy_from(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c'}) status_list = (200, 200) # acct cont with self.controller_context(req, *status_list) as controller: try: controller.PUT(req) except HTTPException as resp: self.assertEquals(resp.status_int // 100, 4) # client error else: raise self.fail('Invalid X-Copy-From did not raise ' 'client error') def test_copy_server_error_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) status_list = (200, 200, 503, 503, 503) # acct cont objc objc objc with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 503) def test_copy_not_found_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) # not found status_list = (200, 200, 404, 404, 404) # acct cont objc objc objc with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) def test_copy_with_some_missing_sources(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) status_list = (200, 200, 404, 404, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) def test_copy_with_object_metadata(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o', 'X-Object-Meta-Ours': 'okay'}) # test object metadata status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') def test_copy_source_larger_than_max_file_size(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) # copy-from object is too large to fit in target object class LargeResponseBody(object): def __len__(self): return MAX_FILE_SIZE + 1 def __getitem__(self, key): return '' copy_from_obj_body = LargeResponseBody() status_list = (200, 200, 200, 200, 200) # acct cont objc objc objc kwargs = dict(body=copy_from_obj_body) with self.controller_context(req, *status_list, **kwargs) as controller: self.app.update_request(req) self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 413) def test_basic_COPY(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c/o2'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') def test_COPY_across_containers(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c2/o'}) status_list = (200, 200, 200, 200, 200, 200, 201, 201, 201) # acct cont c2 objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') def test_COPY_source_with_slashes_in_name(self): req = Request.blank('/v1/a/c/o/o2', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c/o'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') def test_COPY_destination_leading_slash(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o') def test_COPY_source_with_slashes_destination_leading_slash(self): req = Request.blank('/v1/a/c/o/o2', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from'], 'c/o/o2') def test_COPY_no_object_in_destination(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c_o'}) status_list = [] # no requests needed with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 412) def test_COPY_server_error_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) status_list = (200, 200, 503, 503, 503) # acct cont objc objc objc with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 503) def test_COPY_not_found_reading_source(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) status_list = (200, 200, 404, 404, 404) # acct cont objc objc objc with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 404) def test_COPY_with_some_missing_sources(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) status_list = (200, 200, 404, 404, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) def test_COPY_with_metadata(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o', 'X-Object-Meta-Ours': 'okay'}) status_list = (200, 200, 200, 200, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj with self.controller_context(req, *status_list) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers.get('x-object-meta-test'), 'testing') self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') self.assertEquals(resp.headers.get('x-delete-at'), '9876543210') def test_COPY_source_larger_than_max_file_size(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) class LargeResponseBody(object): def __len__(self): return MAX_FILE_SIZE + 1 def __getitem__(self, key): return '' copy_from_obj_body = LargeResponseBody() status_list = (200, 200, 200, 200, 200) # acct cont objc objc objc kwargs = dict(body=copy_from_obj_body) with self.controller_context(req, *status_list, **kwargs) as controller: resp = controller.COPY(req) self.assertEquals(resp.status_int, 413) def test_COPY_newest(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) req.account = 'a' controller.object_name = 'o' set_http_connect(200, 200, 200, 200, 200, 201, 201, 201, #act cont objc objc objc obj obj obj timestamps=('1', '1', '1', '3', '2', '4', '4', '4')) self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) self.assertEquals(resp.headers['x-copied-from-last-modified'], '3') def test_COPY_delete_at(self): with save_globals(): given_headers = {} def fake_connect_put_node(nodes, part, path, headers, logger_thread_locals): given_headers.update(headers) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') controller._connect_put_node = fake_connect_put_node set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': '/c/o'}) self.app.update_request(req) controller.COPY(req) self.assertEquals(given_headers.get('X-Delete-At'), '9876543210') self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) def test_chunked_put(self): class ChunkedFile(object): def __init__(self, bytes): self.bytes = bytes self.read_bytes = 0 @property def bytes_left(self): return self.bytes - self.read_bytes def read(self, amt=None): if self.read_bytes >= self.bytes: raise StopIteration() if not amt: amt = self.bytes_left data = 'a' * min(amt, self.bytes_left) self.read_bytes += len(data) return data with save_globals(): set_http_connect(201, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Transfer-Encoding': 'chunked', 'Content-Type': 'foo/bar'}) req.body_file = ChunkedFile(10) self.app.memcache.store = {} self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int // 100, 2) # success # test 413 entity to large set_http_connect(201, 201, 201, 201) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Transfer-Encoding': 'chunked', 'Content-Type': 'foo/bar'}) req.body_file = ChunkedFile(11) self.app.memcache.store = {} self.app.update_request(req) try: swift.proxy.controllers.obj.MAX_FILE_SIZE = 10 res = controller.PUT(req) self.assertEquals(res.status_int, 413) finally: swift.proxy.controllers.obj.MAX_FILE_SIZE = MAX_FILE_SIZE def test_chunked_put_bad_version(self): # Check bad version (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) def test_chunked_put_bad_path(self): # Check bad path (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) def test_chunked_put_bad_utf8(self): # Check invalid utf-8 (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) def test_chunked_put_bad_path_no_controller(self): # Check bad path, no controller (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) def test_chunked_put_bad_method(self): # Check bad method (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 405' self.assertEquals(headers[:len(exp)], exp) def test_chunked_put_unhandled_exception(self): # Check unhandled exception (prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv) = _test_servers (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets orig_update_request = prosrv.update_request def broken_update_request(*args, **kwargs): raise Exception('fake: this should be printed') prosrv.update_request = broken_update_request sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 500' self.assertEquals(headers[:len(exp)], exp) prosrv.update_request = orig_update_request def test_chunked_put_head_account(self): # Head account, just a double check and really is here to test # the part Application.log_request that 'enforces' a # content_length on the response. (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 204' self.assertEquals(headers[:len(exp)], exp) self.assert_('\r\nContent-Length: 0\r\n' in headers) def test_chunked_put_utf8_all_the_way_down(self): # Test UTF-8 Unicode all the way through the system ustr = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xba \xe1\xbc\xb0\xce' \ '\xbf\xe1\xbd\xbb\xce\x87 \xcf\x84\xe1\xbd\xb0 \xcf' \ '\x80\xe1\xbd\xb1\xce\xbd\xcf\x84\xca\xbc \xe1\xbc' \ '\x82\xce\xbd \xe1\xbc\x90\xce\xbe\xe1\xbd\xb5\xce' \ '\xba\xce\xbf\xce\xb9 \xcf\x83\xce\xb1\xcf\x86\xe1' \ '\xbf\x86.Test' ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest' # Create ustr container (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n' % quote(ustr)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # List account with ustr container (test plain) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) containers = fd.read().split('\n') self.assert_(ustr in containers) # List account with ustr container (test json) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a?format=json HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) listing = simplejson.loads(fd.read()) self.assert_(ustr.decode('utf8') in [l['name'] for l in listing]) # List account with ustr container (test xml) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a?format=xml HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('%s' % ustr in fd.read()) # Create ustr object with ustr metadata in ustr container sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'X-Object-Meta-%s: %s\r\nContent-Length: 0\r\n\r\n' % (quote(ustr), quote(ustr), quote(ustr_short), quote(ustr))) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # List ustr container with ustr object (test plain) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n' % quote(ustr)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) objects = fd.read().split('\n') self.assert_(ustr in objects) # List ustr container with ustr object (test json) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?format=json HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' % quote(ustr)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) listing = simplejson.loads(fd.read()) self.assertEquals(listing[0]['name'], ustr.decode('utf8')) # List ustr container with ustr object (test xml) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?format=xml HTTP/1.1\r\n' 'Host: localhost\r\nConnection: close\r\n' 'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' % quote(ustr)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('%s' % ustr in fd.read()) # Retrieve ustr object with ustr metadata sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n' % (quote(ustr), quote(ustr))) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('\r\nX-Object-Meta-%s: %s\r\n' % (quote(ustr_short).lower(), quote(ustr)) in headers) def test_chunked_put_chunked_put(self): # Do chunked object put (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() # Also happens to assert that x-storage-token is taken as a # replacement for x-auth-token. fd.write('PUT /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Transfer-Encoding: chunked\r\n\r\n' '2\r\noh\r\n4\r\n hai\r\nf\r\n123456789abcdef\r\n' '0\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Ensure we get what we put sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/c/o/chunky HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) body = fd.read() self.assertEquals(body, 'oh hai123456789abcdef') def test_version_manifest(self, oc='versions', vc='vers', o='name'): versions_to_create = 3 # Create a container for our versioned object testing (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() pre = quote('%03x' % len(o)) osub = '%s/sub' % o presub = quote('%03x' % len(osub)) osub = quote(osub) presub = quote(presub) oc = quote(oc) vc = quote(vc) fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\nX-Versions-Location: %s\r\n\r\n' % (oc, vc)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # check that the header was set sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) self.assert_('X-Versions-Location: %s' % vc in headers) # make the container for the object versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n' % vc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Create the versioned file sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' 'X-Object-Meta-Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Create the object versions for segment in xrange(1, versions_to_create): sleep(.01) # guarantee that the timestamp changes sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish%s' '\r\n\r\n%05d\r\n' % (oc, o, segment, segment)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Ensure retrieving the manifest file gets the latest version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n' '\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('Content-Type: text/jibberish%s' % segment in headers) self.assert_('X-Object-Meta-Foo: barbaz' not in headers) body = fd.read() self.assertEquals(body, '%05d' % segment) # Ensure we have the right number of versions saved sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) body = fd.read() versions = [x for x in body.split('\n') if x] self.assertEquals(len(versions), versions_to_create - 1) # copy a version and make sure the version info is stripped sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('COPY /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: ' 't\r\nDestination: %s/copied_name\r\n' 'Content-Length: 0\r\n\r\n' % (oc, o, oc)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response to the COPY self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/copied_name HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) body = fd.read() self.assertEquals(body, '%05d' % segment) # post and make sure it's updated sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('POST /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: ' 't\r\nContent-Type: foo/bar\r\nContent-Length: 0\r\n' 'X-Object-Meta-Bar: foo\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response to the POST self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('Content-Type: foo/bar' in headers) self.assert_('X-Object-Meta-Bar: foo' in headers) body = fd.read() self.assertEquals(body, '%05d' % segment) # Delete the object versions for segment in xrange(versions_to_create - 1, 0, -1): sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r' '\nConnection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) # Ensure retrieving the manifest file gets the latest version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Auth-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) self.assert_('Content-Type: text/jibberish%s' % (segment - 1) in headers) body = fd.read() self.assertEquals(body, '%05d' % (segment - 1)) # Ensure we have the right number of versions saved sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r' '\n' % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) body = fd.read() versions = [x for x in body.split('\n') if x] self.assertEquals(len(versions), segment - 1) # there is now one segment left (in the manifest) # Ensure we have no saved versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 204 No Content' self.assertEquals(headers[:len(exp)], exp) # delete the last verision sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) # Ensure it's all gone sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 404' self.assertEquals(headers[:len(exp)], exp) # make sure manifest files don't get versioned sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 0\r\nContent-Type: text/jibberish0\r\n' 'Foo: barbaz\r\nX-Object-Manifest: %s/foo_\r\n\r\n' % (oc, vc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Ensure we have no saved versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (vc, pre, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 204 No Content' self.assertEquals(headers[:len(exp)], exp) # DELETE v1/a/c/obj shouldn't delete v1/a/c/obj/sub versions sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' 'Foo: barbaz\r\n\r\n00000\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\nContent-Type: text/jibberish0\r\n' 'Foo: barbaz\r\n\r\n00001\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' 'Foo: barbaz\r\n\r\nsub1\r\n' % (oc, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%s/%s HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 4\r\nContent-Type: text/jibberish0\r\n' 'Foo: barbaz\r\n\r\nsub2\r\n' % (oc, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('DELETE /v1/a/%s/%s HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % (oc, o)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/%s?prefix=%s%s/ HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n\r\n' % (vc, presub, osub)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx series response self.assertEquals(headers[:len(exp)], exp) body = fd.read() versions = [x for x in body.split('\n') if x] self.assertEquals(len(versions), 1) # Check for when the versions target container doesn't exist sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%swhoops HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\nX-Versions-Location: none\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Create the versioned file sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\n\r\n00000\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) # Create another version sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/%swhoops/foo HTTP/1.1\r\nHost: ' 'localhost\r\nConnection: close\r\nX-Storage-Token: ' 't\r\nContent-Length: 5\r\n\r\n00001\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 412' self.assertEquals(headers[:len(exp)], exp) # Delete the object sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('DELETE /v1/a/%swhoops/foo HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n\r\n' % oc) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 2' # 2xx response self.assertEquals(headers[:len(exp)], exp) def test_version_manifest_utf8(self): oc = '0_oc_non_ascii\xc2\xa3' vc = '0_vc_non_ascii\xc2\xa3' o = '0_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_container(self): oc = '1_oc_non_ascii\xc2\xa3' vc = '1_vc_ascii' o = '1_o_ascii' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_version_container(self): oc = '2_oc_ascii' vc = '2_vc_non_ascii\xc2\xa3' o = '2_o_ascii' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_containers(self): oc = '3_oc_non_ascii\xc2\xa3' vc = '3_vc_non_ascii\xc2\xa3' o = '3_o_ascii' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_object(self): oc = '4_oc_ascii' vc = '4_vc_ascii' o = '4_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_version_container_utf_object(self): oc = '5_oc_ascii' vc = '5_vc_non_ascii\xc2\xa3' o = '5_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) def test_version_manifest_utf8_container_utf_object(self): oc = '6_oc_non_ascii\xc2\xa3' vc = '6_vc_ascii' o = '6_o_non_ascii\xc2\xa3' self.test_version_manifest(oc, vc, o) def test_conditional_range_get(self): (prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \ _test_sockets sock = connect_tcp(('localhost', prolis.getsockname()[1])) # make a container fd = sock.makefile() fd.write('PUT /v1/a/con HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\nX-Storage-Token: t\r\n' 'Content-Length: 0\r\n\r\n') fd.flush() exp = 'HTTP/1.1 201' headers = readuntil2crlfs(fd) self.assertEquals(headers[:len(exp)], exp) # put an object in it sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/con/o HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Length: 10\r\n' 'Content-Type: text/plain\r\n' '\r\n' 'abcdefghij\r\n') fd.flush() exp = 'HTTP/1.1 201' headers = readuntil2crlfs(fd) self.assertEquals(headers[:len(exp)], exp) # request with both If-None-Match and Range etag = md5("abcdefghij").hexdigest() sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/con/o HTTP/1.1\r\n' + 'Host: localhost\r\n' + 'Connection: close\r\n' + 'X-Storage-Token: t\r\n' + 'If-None-Match: "' + etag + '"\r\n' + 'Range: bytes=3-8\r\n' + '\r\n') fd.flush() exp = 'HTTP/1.1 304' headers = readuntil2crlfs(fd) self.assertEquals(headers[:len(exp)], exp) def test_mismatched_etags(self): with save_globals(): # no etag supplied, object servers return success w/ diff values controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) self.app.update_request(req) set_http_connect(200, 201, 201, 201, etags=[None, '68b329da9893e34099c7d8ad5cb9c940', '68b329da9893e34099c7d8ad5cb9c940', '68b329da9893e34099c7d8ad5cb9c941']) resp = controller.PUT(req) self.assertEquals(resp.status_int // 100, 5) # server error # req supplies etag, object servers return 422 - mismatch req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'Content-Length': '0', 'ETag': '68b329da9893e34099c7d8ad5cb9c940', }) self.app.update_request(req) set_http_connect(200, 422, 422, 503, etags=['68b329da9893e34099c7d8ad5cb9c940', '68b329da9893e34099c7d8ad5cb9c941', None, None]) resp = controller.PUT(req) self.assertEquals(resp.status_int // 100, 4) # client error def test_response_get_accept_ranges_header(self): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.GET(req) self.assert_('accept-ranges' in resp.headers) self.assertEquals(resp.headers['accept-ranges'], 'bytes') def test_response_head_accept_ranges_header(self): with save_globals(): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200) resp = controller.HEAD(req) self.assert_('accept-ranges' in resp.headers) self.assertEquals(resp.headers['accept-ranges'], 'bytes') def test_GET_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o') req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.GET(req) self.assert_(called[0]) def test_HEAD_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.HEAD(req) self.assert_(called[0]) def test_POST_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): self.app.object_post_as_copy = False set_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.POST(req) self.assert_(called[0]) def test_POST_as_copy_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Length': '5'}, body='12345') req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.POST(req) self.assert_(called[0]) def test_PUT_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.PUT(req) self.assert_(called[0]) def test_COPY_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 200, 200, 200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, headers={'Destination': 'c/o'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.COPY(req) self.assert_(called[0]) def test_POST_converts_delete_after_to_delete_at(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) self.app.memcache.store = {} orig_time = time.time try: t = time.time() time.time = lambda: t req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60'}) self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status, '202 Fake') self.assertEquals(req.headers.get('x-delete-at'), str(int(t + 60))) self.app.object_post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 202, 202, 202) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60'}) self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status, '202 Fake') self.assertEquals(req.headers.get('x-delete-at'), str(int(t + 60))) finally: time.time = orig_time def test_POST_non_int_delete_after(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-After': '60.1'}) self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status, '400 Bad Request') self.assertTrue('Non-integer X-Delete-After' in res.body) def test_POST_negative_delete_after(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 200, 200, 200, 202, 202, 202) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-After': '-60'}) self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status, '400 Bad Request') self.assertTrue('X-Delete-At in past' in res.body) def test_POST_delete_at(self): with save_globals(): given_headers = {} def fake_make_requests(req, ring, part, method, path, headers, query_string=''): given_headers.update(headers[0]) self.app.object_post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') controller.make_requests = fake_make_requests set_http_connect(200, 200) self.app.memcache.store = {} t = str(int(time.time() + 100)) req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) controller.POST(req) self.assertEquals(given_headers.get('X-Delete-At'), t) self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) t = str(int(time.time() + 100)) + '.1' req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) self.assertTrue('Non-integer X-Delete-At' in resp.body) t = str(int(time.time() - 100)) req = Request.blank('/v1/a/c/o', {}, headers={'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) self.assertTrue('X-Delete-At in past' in resp.body) def test_PUT_converts_delete_after_to_delete_at(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) self.app.memcache.store = {} orig_time = time.time try: t = time.time() time.time = lambda: t req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-After': '60'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status, '201 Fake') self.assertEquals(req.headers.get('x-delete-at'), str(int(t + 60))) finally: time.time = orig_time def test_PUT_non_int_delete_after(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-After': '60.1'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status, '400 Bad Request') self.assertTrue('Non-integer X-Delete-After' in res.body) def test_PUT_negative_delete_after(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') set_http_connect(200, 200, 201, 201, 201) self.app.memcache.store = {} req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-After': '-60'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status, '400 Bad Request') self.assertTrue('X-Delete-At in past' in res.body) def test_PUT_delete_at(self): with save_globals(): given_headers = {} def fake_connect_put_node(nodes, part, path, headers, logger_thread_locals): given_headers.update(headers) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') controller._connect_put_node = fake_connect_put_node set_http_connect(200, 200) self.app.memcache.store = {} t = str(int(time.time() + 100)) req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) controller.PUT(req) self.assertEquals(given_headers.get('X-Delete-At'), t) self.assertTrue('X-Delete-At-Host' in given_headers) self.assertTrue('X-Delete-At-Device' in given_headers) self.assertTrue('X-Delete-At-Partition' in given_headers) self.assertTrue('X-Delete-At-Container' in given_headers) t = str(int(time.time() + 100)) + '.1' req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) self.assertTrue('Non-integer X-Delete-At' in resp.body) t = str(int(time.time() - 100)) req = Request.blank('/v1/a/c/o', {}, headers={'Content-Length': '0', 'Content-Type': 'foo/bar', 'X-Delete-At': t}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) self.assertTrue('X-Delete-At in past' in resp.body) def test_leak_1(self): _request_instances = weakref.WeakKeyDictionary() _orig_init = Request.__init__ def request_init(self, *args, **kwargs): _orig_init(self, *args, **kwargs) _request_instances[self] = None with mock.patch.object(Request, "__init__", request_init): prolis = _test_sockets[0] prosrv = _test_servers[0] obj_len = prosrv.client_chunk_size * 2 # PUT test file sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Auth-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' '\r\n%s' % (obj_len, 'a' * obj_len)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) # Remember Request instance count, make sure the GC is run for # pythons without reference counting. for i in xrange(4): sleep(0) # let eventlet do its thing gc.collect() else: sleep(0) before_request_instances = len(_request_instances) # GET test file, but disconnect early sock = connect_tcp(('localhost', prolis.getsockname()[1])) fd = sock.makefile() fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Auth-Token: t\r\n' '\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEqual(headers[:len(exp)], exp) fd.read(1) fd.close() sock.close() # Make sure the GC is run again for pythons without reference # counting for i in xrange(4): sleep(0) # let eventlet do its thing gc.collect() else: sleep(0) self.assertEquals( before_request_instances, len(_request_instances)) def test_OPTIONS(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o.jpg') def my_empty_container_info(*args): return {} controller.container_info = my_empty_container_info req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) def my_empty_origin_container_info(*args): return {'cors': {'allow_origin': None}} controller.container_info = my_empty_origin_container_info req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) def my_container_info(*args): return { 'cors': { 'allow_origin': 'http://foo.bar:8080 https://foo.bar', 'max_age': '999', } } controller.container_info = my_container_info req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://foo.bar', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) self.assertEquals( 'https://foo.bar', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEquals( len(resp.headers['access-control-allow-methods'].split(', ')), 7) self.assertEquals('999', resp.headers['access-control-max-age']) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://foo.bar'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) req = Request.blank('/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) self.assertEquals(len(resp.headers['Allow'].split(', ')), 7) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.bar', 'Access-Control-Request-Method': 'GET'}) controller.app.cors_allow_origin = ['http://foo.bar', ] resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) def my_container_info_wildcard(*args): return { 'cors': { 'allow_origin': '*', 'max_age': '999', } } controller.container_info = my_container_info_wildcard req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://bar.baz', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS COPY GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEquals( len(resp.headers['access-control-allow-methods'].split(', ')), 7) self.assertEquals('999', resp.headers['access-control-max-age']) def test_CORS_valid(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { 'cors': { 'allow_origin': 'http://not.foo.bar' } } controller.container_info = stubContainerInfo controller.app.strict_cors_mode = False def objectGET(controller, req): return Response(headers={ 'X-Object-Meta-Color': 'red', 'X-Super-Secret': 'hush', }) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'GET'}, headers={'Origin': 'http://foo.bar'}) resp = cors_validation(objectGET)(controller, req) self.assertEquals(200, resp.status_int) self.assertEquals('http://foo.bar', resp.headers['access-control-allow-origin']) self.assertEquals('red', resp.headers['x-object-meta-color']) # X-Super-Secret is in the response, but not "exposed" self.assertEquals('hush', resp.headers['x-super-secret']) self.assertTrue('access-control-expose-headers' in resp.headers) exposed = set( h.strip() for h in resp.headers['access-control-expose-headers'].split(',')) expected_exposed = set(['cache-control', 'content-language', 'content-type', 'expires', 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id', 'x-object-meta-color']) self.assertEquals(expected_exposed, exposed) controller.app.strict_cors_mode = True req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'GET'}, headers={'Origin': 'http://foo.bar'}) resp = cors_validation(objectGET)(controller, req) self.assertEquals(200, resp.status_int) self.assertTrue('access-control-allow-origin' not in resp.headers) def test_CORS_valid_with_obj_headers(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') def stubContainerInfo(*args): return { 'cors': { 'allow_origin': 'http://foo.bar' } } controller.container_info = stubContainerInfo def objectGET(controller, req): return Response(headers={ 'X-Object-Meta-Color': 'red', 'X-Super-Secret': 'hush', 'Access-Control-Allow-Origin': 'http://obj.origin', 'Access-Control-Expose-Headers': 'x-trans-id' }) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'GET'}, headers={'Origin': 'http://foo.bar'}) resp = cors_validation(objectGET)(controller, req) self.assertEquals(200, resp.status_int) self.assertEquals('http://obj.origin', resp.headers['access-control-allow-origin']) self.assertEquals('x-trans-id', resp.headers['access-control-expose-headers']) def _gather_x_container_headers(self, controller_call, req, *connect_args, **kwargs): header_list = kwargs.pop('header_list', ['X-Container-Device', 'X-Container-Host', 'X-Container-Partition']) seen_headers = [] def capture_headers(ipaddr, port, device, partition, method, path, headers=None, query_string=None): captured = {} for header in header_list: captured[header] = headers.get(header) seen_headers.append(captured) with save_globals(): self.app.allow_account_management = True set_http_connect(*connect_args, give_connect=capture_headers, **kwargs) resp = controller_call(req) self.assertEqual(2, resp.status_int // 100) # sanity check # don't care about the account/container HEADs, so chuck # the first two requests return sorted(seen_headers[2:], key=lambda d: d.get(header_list[0]) or 'z') def test_PUT_x_container_headers_with_equal_replicas(self): req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_PUT_x_container_headers_with_fewer_container_replicas(self): self.app.container_ring.set_replicas(2) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': None, 'X-Container-Partition': None, 'X-Container-Device': None}]) def test_PUT_x_container_headers_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '5'}, body='12345') controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_POST_x_container_headers_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) self.app.object_post_as_copy = False req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'application/stuff'}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.POST, req, 200, 200, 200, 200, 200) # HEAD HEAD POST POST POST self.assertEqual( seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_DELETE_x_container_headers_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'Content-Type': 'application/stuff'}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.DELETE, req, 200, 200, 200, 200, 200) # HEAD HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'} ]) @mock.patch('time.time', new=lambda: STATIC_TIME) def test_PUT_x_delete_at_with_fewer_container_replicas(self): self.app.container_ring.set_replicas(2) delete_at_timestamp = int(time.time()) + 100000 delete_at_container = str( delete_at_timestamp / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': '0', 'X-Delete-At': str(delete_at_timestamp)}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT header_list=('X-Delete-At-Host', 'X-Delete-At-Device', 'X-Delete-At-Partition', 'X-Delete-At-Container')) self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000', 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda'}, {'X-Delete-At-Host': '10.0.0.1:1001', 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': None, 'X-Delete-At-Container': None, 'X-Delete-At-Partition': None, 'X-Delete-At-Device': None} ]) @mock.patch('time.time', new=lambda: STATIC_TIME) def test_PUT_x_delete_at_with_more_container_replicas(self): self.app.container_ring.set_replicas(4) self.app.expiring_objects_account = 'expires' self.app.expiring_objects_container_divisor = 60 delete_at_timestamp = int(time.time()) + 100000 delete_at_container = str( delete_at_timestamp / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Type': 'application/stuff', 'Content-Length': 0, 'X-Delete-At': str(delete_at_timestamp)}) controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') seen_headers = self._gather_x_container_headers( controller.PUT, req, 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT header_list=('X-Delete-At-Host', 'X-Delete-At-Device', 'X-Delete-At-Partition', 'X-Delete-At-Container')) self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda,sdd'}, {'X-Delete-At-Host': '10.0.0.1:1001', 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': '10.0.0.2:1002', 'X-Delete-At-Container': delete_at_container, 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdc'} ]) class TestContainerController(unittest.TestCase): "Test swift.proxy_server.ContainerController" def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing(), logger=FakeLogger()) def test_transfer_headers(self): src_headers = {'x-remove-versions-location': 'x', 'x-container-read': '*:user'} dst_headers = {'x-versions-location': 'backup'} controller = swift.proxy.controllers.ContainerController(self.app, 'a', 'c') controller.transfer_headers(src_headers, dst_headers) expected_headers = {'x-versions-location': '', 'x-container-read': '*:user'} self.assertEqual(dst_headers, expected_headers) def assert_status_map(self, method, statuses, expected, raise_exc=False, missing_container=False): with save_globals(): kwargs = {} if raise_exc: kwargs['raise_exc'] = raise_exc kwargs['missing_container'] = missing_container set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c/', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) def test_HEAD_GET(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') def test_status_map(statuses, expected, c_expected=None, a_expected=None, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) self.app.update_request(req) res = controller.HEAD(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) if expected < 400: self.assert_('x-works' in res.headers) self.assertEquals(res.headers['x-works'], 'yes') if c_expected: self.assertTrue('swift.container/a/c' in res.environ) self.assertEquals( res.environ['swift.container/a/c']['status'], c_expected) else: self.assertTrue('swift.container/a/c' not in res.environ) if a_expected: self.assertTrue('swift.account/a' in res.environ) self.assertEquals(res.environ['swift.account/a']['status'], a_expected) else: self.assertTrue('swift.account/a' not in res.environ) set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) self.app.update_request(req) res = controller.GET(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) if expected < 400: self.assert_('x-works' in res.headers) self.assertEquals(res.headers['x-works'], 'yes') if c_expected: self.assertTrue('swift.container/a/c' in res.environ) self.assertEquals( res.environ['swift.container/a/c']['status'], c_expected) else: self.assertTrue('swift.container/a/c' not in res.environ) if a_expected: self.assertTrue('swift.account/a' in res.environ) self.assertEquals(res.environ['swift.account/a']['status'], a_expected) else: self.assertTrue('swift.account/a' not in res.environ) # In all the following tests cache 200 for account # return and ache vary for container # return 200 and cache 200 for and container test_status_map((200, 200, 404, 404), 200, 200, 200) test_status_map((200, 200, 500, 404), 200, 200, 200) # return 304 dont cache container test_status_map((200, 304, 500, 404), 304, None, 200) # return 404 and cache 404 for container test_status_map((200, 404, 404, 404), 404, 404, 200) test_status_map((200, 404, 404, 500), 404, 404, 200) # return 503, dont cache container test_status_map((200, 500, 500, 500), 503, None, 200) self.assertFalse(self.app.account_autocreate) # In all the following tests cache 404 for account # return 404 (as account is not found) and dont cache container test_status_map((404, 404, 404), 404, None, 404) # This should make no difference self.app.account_autocreate = True test_status_map((404, 404, 404), 404, None, 404) def test_PUT(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) req.content_length = 0 self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 201, 201, 201), 201, missing_container=True) test_status_map((200, 201, 201, 500), 201, missing_container=True) test_status_map((200, 204, 404, 404), 404, missing_container=True) test_status_map((200, 204, 500, 404), 503, missing_container=True) self.assertFalse(self.app.account_autocreate) test_status_map((404, 404, 404), 404, missing_container=True) self.app.account_autocreate = True # fail to retrieve account info test_status_map( (503, 503, 503), # account_info fails on 503 404, missing_container=True) # account fail after creation test_status_map( (404, 404, 404, # account_info fails on 404 201, 201, 201, # PUT account 404, 404, 404), # account_info fail 404, missing_container=True) test_status_map( (503, 503, 404, # account_info fails on 404 503, 503, 503, # PUT account 503, 503, 404), # account_info fail 404, missing_container=True) # put fails test_status_map( (404, 404, 404, # account_info fails on 404 201, 201, 201, # PUT account 200, # account_info success 503, 503, 201), # put container fail 503, missing_container=True) # all goes according to plan test_status_map( (404, 404, 404, # account_info fails on 404 201, 201, 201, # PUT account 200, # account_info success 201, 201, 201), # put container success 201, missing_container=True) test_status_map( (503, 404, 404, # account_info fails on 404 503, 201, 201, # PUT account 503, 200, # account_info success 503, 201, 201), # put container success 201, missing_container=True) def test_POST(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a/c', {}) req.content_length = 0 self.app.update_request(req) res = controller.POST(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((200, 201, 201, 201), 201, missing_container=True) test_status_map((200, 201, 201, 500), 201, missing_container=True) test_status_map((200, 204, 404, 404), 404, missing_container=True) test_status_map((200, 204, 500, 404), 503, missing_container=True) self.assertFalse(self.app.account_autocreate) test_status_map((404, 404, 404), 404, missing_container=True) self.app.account_autocreate = True test_status_map((404, 404, 404), 404, missing_container=True) def test_PUT_max_containers_per_account(self): with save_globals(): self.app.max_containers_per_account = 12346 controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.PUT, (200, 201, 201, 201), 201, missing_container=True) self.app.max_containers_per_account = 12345 controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.PUT, (201, 201, 201), 403, missing_container=True) self.app.max_containers_per_account = 12345 self.app.max_containers_whitelist = ['account'] controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.PUT, (200, 201, 201, 201), 201, missing_container=True) def test_PUT_max_container_name_length(self): with save_globals(): limit = MAX_CONTAINER_NAME_LENGTH controller = proxy_server.ContainerController(self.app, 'account', '1' * limit) self.assert_status_map(controller.PUT, (200, 201, 201, 201), 201, missing_container=True) controller = proxy_server.ContainerController(self.app, 'account', '2' * (limit + 1)) self.assert_status_map(controller.PUT, (201, 201, 201), 400, missing_container=True) def test_PUT_connect_exceptions(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.PUT, (200, 201, 201, -1), 201, missing_container=True) self.assert_status_map(controller.PUT, (200, 201, -1, -1), 503, missing_container=True) self.assert_status_map(controller.PUT, (200, 503, 503, -1), 503, missing_container=True) def test_acc_missing_returns_404(self): for meth in ('DELETE', 'PUT'): with save_globals(): self.app.memcache = FakeMemcacheReturnsNone() for dev in self.app.account_ring.devs.values(): del dev['errors'] del dev['last_error'] controller = proxy_server.ContainerController(self.app, 'account', 'container') if meth == 'PUT': set_http_connect(200, 200, 200, 200, 200, 200, missing_container=True) else: set_http_connect(200, 200, 200, 200) self.app.memcache.store = {} req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) self.app.update_request(req) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 200) set_http_connect(404, 404, 404, 200, 200, 200) # Make sure it is a blank request wthout env caching req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, 404) # Make sure it is a blank request wthout env caching req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) set_http_connect(503, 404, raise_exc=True) # Make sure it is a blank request wthout env caching req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) for dev in self.app.account_ring.devs.values(): dev['errors'] = self.app.error_suppression_limit + 1 dev['last_error'] = time.time() set_http_connect(200, 200, 200, 200, 200, 200) # Make sure it is a blank request wthout env caching req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': meth}) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 404) def test_put_locking(self): class MockMemcache(FakeMemcache): def __init__(self, allow_lock=None): self.allow_lock = allow_lock super(MockMemcache, self).__init__() @contextmanager def soft_lock(self, key, timeout=0, retries=5): if self.allow_lock: yield True else: raise NotImplementedError with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') self.app.memcache = MockMemcache(allow_lock=True) set_http_connect(200, 201, 201, 201, missing_container=True) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 201) def test_error_limiting(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') controller.app.sort_nodes = lambda l: l self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, missing_container=False) self.assertEquals( controller.app.container_ring.devs[0]['errors'], 2) self.assert_('last_error' in controller.app.container_ring.devs[0]) for _junk in xrange(self.app.error_suppression_limit): self.assert_status_map(controller.HEAD, (200, 503, 503, 503), 503) self.assertEquals(controller.app.container_ring.devs[0]['errors'], self.app.error_suppression_limit + 1) self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503) self.assert_('last_error' in controller.app.container_ring.devs[0]) self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503, missing_container=True) self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 503) self.app.error_suppression_interval = -300 self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200) self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 404, raise_exc=True) def test_DELETE(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 204) self.assert_status_map(controller.DELETE, (200, 204, 204, 503), 204) self.assert_status_map(controller.DELETE, (200, 204, 503, 503), 503) self.assert_status_map(controller.DELETE, (200, 204, 404, 404), 404) self.assert_status_map(controller.DELETE, (200, 404, 404, 404), 404) self.assert_status_map(controller.DELETE, (200, 204, 503, 404), 503) self.app.memcache = FakeMemcacheReturnsNone() # 200: Account check, 404x3: Container check self.assert_status_map(controller.DELETE, (200, 404, 404, 404), 404) def test_response_get_accept_ranges_header(self): with save_globals(): set_http_connect(200, 200, body='{}') controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c?format=json') self.app.update_request(req) res = controller.GET(req) self.assert_('accept-ranges' in res.headers) self.assertEqual(res.headers['accept-ranges'], 'bytes') def test_response_head_accept_ranges_header(self): with save_globals(): set_http_connect(200, 200, body='{}') controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c?format=json') self.app.update_request(req) res = controller.HEAD(req) self.assert_('accept-ranges' in res.headers) self.assertEqual(res.headers['accept-ranges'], 'bytes') def test_PUT_metadata(self): self.metadata_helper('PUT') def test_POST_metadata(self): self.metadata_helper('POST') def metadata_helper(self, method): for test_header, test_value in ( ('X-Container-Meta-TestHeader', 'TestValue'), ('X-Container-Meta-TestHeader', ''), ('X-Remove-Container-Meta-TestHeader', 'anything'), ('X-Container-Read', '.r:*'), ('X-Remove-Container-Read', 'anything'), ('X-Container-Write', 'anyone'), ('X-Remove-Container-Write', 'anything')): test_errors = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a/c': find_header = test_header find_value = test_value if find_header.lower().startswith('x-remove-'): find_header = \ find_header.lower().replace('-remove', '', 1) find_value = '' for k, v in headers.iteritems(): if k.lower() == find_header.lower() and \ v == find_value: break else: test_errors.append('%s: %s not in %s' % (find_header, find_value, headers)) with save_globals(): controller = \ proxy_server.ContainerController(self.app, 'a', 'c') set_http_connect(200, 201, 201, 201, give_connect=test_connect) req = Request.blank( '/v1/a/c', environ={'REQUEST_METHOD': method, 'swift_owner': True}, headers={test_header: test_value}) self.app.update_request(req) getattr(controller, method)(req) self.assertEquals(test_errors, []) def test_PUT_bad_metadata(self): self.bad_metadata_helper('PUT') def test_POST_bad_metadata(self): self.bad_metadata_helper('POST') def bad_metadata_helper(self, method): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') set_http_connect(200, 201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} header_value = 'a' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Container-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Container-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Container-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) def test_POST_calls_clean_acl(self): called = [False] def clean_acl(header, value): called[0] = True raise ValueError('fake error') with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Container-Read': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) controller.POST(req) self.assert_(called[0]) called[0] = False with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Container-Write': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) controller.POST(req) self.assert_(called[0]) def test_PUT_calls_clean_acl(self): called = [False] def clean_acl(header, value): called[0] = True raise ValueError('fake error') with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Container-Read': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) controller.PUT(req) self.assert_(called[0]) called[0] = False with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Container-Write': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) controller.PUT(req) self.assert_(called[0]) def test_GET_no_content(self): with save_globals(): set_http_connect(200, 204, 204, 204) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c') self.app.update_request(req) res = controller.GET(req) self.assertEquals(res.status_int, 204) self.assertEquals( res.environ['swift.container/a/c']['status'], 204) self.assertEquals(res.content_length, 0) self.assertTrue('transfer-encoding' not in res.headers) def test_GET_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c') req.environ['swift.authorize'] = authorize self.app.update_request(req) res = controller.GET(req) self.assertEquals(res.environ['swift.container/a/c']['status'], 201) self.assert_(called[0]) def test_HEAD_calls_authorize(self): called = [False] def authorize(req): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): set_http_connect(200, 201, 201, 201) controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'HEAD'}) req.environ['swift.authorize'] = authorize self.app.update_request(req) controller.HEAD(req) self.assert_(called[0]) def test_OPTIONS_get_info_drops_origin(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') count = [0] def my_get_info(app, env, account, container=None, ret_not_found=False, swift_source=None): if count[0] > 11: return {} count[0] += 1 if not container: return {'some': 'stuff'} return proxy_base.was_get_info( app, env, account, container, ret_not_found, swift_source) proxy_base.was_get_info = proxy_base.get_info with mock.patch.object(proxy_base, 'get_info', my_get_info): proxy_base.get_info = my_get_info req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) controller.OPTIONS(req) self.assertTrue(count[0] < 11) def test_OPTIONS(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') def my_empty_container_info(*args): return {} controller.container_info = my_empty_container_info req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) def my_empty_origin_container_info(*args): return {'cors': {'allow_origin': None}} controller.container_info = my_empty_origin_container_info req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) def my_container_info(*args): return { 'cors': { 'allow_origin': 'http://foo.bar:8080 https://foo.bar', 'max_age': '999', } } controller.container_info = my_container_info req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://foo.bar', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) self.assertEquals( 'https://foo.bar', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEquals( len(resp.headers['access-control-allow-methods'].split(', ')), 6) self.assertEquals('999', resp.headers['access-control-max-age']) req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://foo.bar'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) self.assertEquals(len(resp.headers['Allow'].split(', ')), 6) req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.bar', 'Access-Control-Request-Method': 'GET'}) resp = controller.OPTIONS(req) self.assertEquals(401, resp.status_int) req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.bar', 'Access-Control-Request-Method': 'GET'}) controller.app.cors_allow_origin = ['http://foo.bar', ] resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) def my_container_info_wildcard(*args): return { 'cors': { 'allow_origin': '*', 'max_age': '999', } } controller.container_info = my_container_info_wildcard req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://bar.baz', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) self.assertEquals('*', resp.headers['access-control-allow-origin']) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['access-control-allow-methods']) self.assertEquals( len(resp.headers['access-control-allow-methods'].split(', ')), 6) self.assertEquals('999', resp.headers['access-control-max-age']) req = Request.blank( '/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'https://bar.baz', 'Access-Control-Request-Headers': 'x-foo, x-bar, x-auth-token', 'Access-Control-Request-Method': 'GET'} ) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) self.assertEquals( sortHeaderNames('x-foo, x-bar, x-auth-token'), sortHeaderNames(resp.headers['access-control-allow-headers'])) def test_CORS_valid(self): with save_globals(): controller = proxy_server.ContainerController(self.app, 'a', 'c') def stubContainerInfo(*args): return { 'cors': { 'allow_origin': 'http://foo.bar' } } controller.container_info = stubContainerInfo def containerGET(controller, req): return Response(headers={ 'X-Container-Meta-Color': 'red', 'X-Super-Secret': 'hush', }) req = Request.blank( '/v1/a/c', {'REQUEST_METHOD': 'GET'}, headers={'Origin': 'http://foo.bar'}) resp = cors_validation(containerGET)(controller, req) self.assertEquals(200, resp.status_int) self.assertEquals('http://foo.bar', resp.headers['access-control-allow-origin']) self.assertEquals('red', resp.headers['x-container-meta-color']) # X-Super-Secret is in the response, but not "exposed" self.assertEquals('hush', resp.headers['x-super-secret']) self.assertTrue('access-control-expose-headers' in resp.headers) exposed = set( h.strip() for h in resp.headers['access-control-expose-headers'].split(',')) expected_exposed = set(['cache-control', 'content-language', 'content-type', 'expires', 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id', 'x-container-meta-color']) self.assertEquals(expected_exposed, exposed) def _gather_x_account_headers(self, controller_call, req, *connect_args, **kwargs): seen_headers = [] to_capture = ('X-Account-Partition', 'X-Account-Host', 'X-Account-Device') def capture_headers(ipaddr, port, device, partition, method, path, headers=None, query_string=None): captured = {} for header in to_capture: captured[header] = headers.get(header) seen_headers.append(captured) with save_globals(): self.app.allow_account_management = True set_http_connect(*connect_args, give_connect=capture_headers, **kwargs) resp = controller_call(req) self.assertEqual(2, resp.status_int // 100) # sanity check # don't care about the account HEAD, so throw away the # first element return sorted(seen_headers[1:], key=lambda d: d['X-Account-Host'] or 'Z') def test_PUT_x_account_headers_with_fewer_account_replicas(self): self.app.account_ring.set_replicas(2) req = Request.blank('/v1/a/c', headers={'': ''}) controller = proxy_server.ContainerController(self.app, 'a', 'c') seen_headers = self._gather_x_account_headers( controller.PUT, req, 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', 'X-Account-Partition': '1', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, 'X-Account-Device': None} ]) def test_PUT_x_account_headers_with_more_account_replicas(self): self.app.account_ring.set_replicas(4) req = Request.blank('/v1/a/c', headers={'': ''}) controller = proxy_server.ContainerController(self.app, 'a', 'c') seen_headers = self._gather_x_account_headers( controller.PUT, req, 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Account-Partition': '1', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', 'X-Account-Partition': '1', 'X-Account-Device': 'sdc'} ]) def test_DELETE_x_account_headers_with_fewer_account_replicas(self): self.app.account_ring.set_replicas(2) req = Request.blank('/v1/a/c', headers={'': ''}) controller = proxy_server.ContainerController(self.app, 'a', 'c') seen_headers = self._gather_x_account_headers( controller.DELETE, req, 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', 'X-Account-Partition': '1', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, 'X-Account-Device': None} ]) def test_DELETE_x_account_headers_with_more_account_replicas(self): self.app.account_ring.set_replicas(4) req = Request.blank('/v1/a/c', headers={'': ''}) controller = proxy_server.ContainerController(self.app, 'a', 'c') seen_headers = self._gather_x_account_headers( controller.DELETE, req, 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', 'X-Account-Partition': '1', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', 'X-Account-Partition': '1', 'X-Account-Device': 'sdc'} ]) def test_PUT_backed_x_timestamp_header(self): timestamps = [] def capture_timestamps(*args, **kwargs): headers = kwargs['headers'] timestamps.append(headers.get('X-Timestamp')) req = Request.blank('/v1/a/c', method='PUT', headers={'': ''}) with save_globals(): new_connect = set_http_connect(200, # account existance check 201, 201, 201, give_connect=capture_timestamps) resp = self.app.handle_request(req) # sanity self.assertRaises(StopIteration, new_connect.code_iter.next) self.assertEqual(2, resp.status_int // 100) timestamps.pop(0) # account existance check self.assertEqual(3, len(timestamps)) for timestamp in timestamps: self.assertEqual(timestamp, timestamps[0]) self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) def test_DELETE_backed_x_timestamp_header(self): timestamps = [] def capture_timestamps(*args, **kwargs): headers = kwargs['headers'] timestamps.append(headers.get('X-Timestamp')) req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''}) self.app.update_request(req) with save_globals(): new_connect = set_http_connect(200, # account existance check 201, 201, 201, give_connect=capture_timestamps) resp = self.app.handle_request(req) # sanity self.assertRaises(StopIteration, new_connect.code_iter.next) self.assertEqual(2, resp.status_int // 100) timestamps.pop(0) # account existance check self.assertEqual(3, len(timestamps)) for timestamp in timestamps: self.assertEqual(timestamp, timestamps[0]) self.assert_(re.match('[0-9]{10}\.[0-9]{5}', timestamp)) def test_node_read_timeout_retry_to_container(self): with save_globals(): req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) self.app.node_timeout = 0.1 set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0]) resp = req.get_response(self.app) got_exc = False try: resp.body except ChunkReadTimeout: got_exc = True self.assert_(got_exc) class TestAccountController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing) def assert_status_map(self, method, statuses, expected, env_expected=None): with save_globals(): set_http_connect(*statuses) req = Request.blank('/v1/a', {}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) if env_expected: self.assertEquals(res.environ['swift.account/a']['status'], env_expected) set_http_connect(*statuses) req = Request.blank('/v1/a/', {}) self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) if env_expected: self.assertEquals(res.environ['swift.account/a']['status'], env_expected) def test_OPTIONS(self): with save_globals(): self.app.allow_account_management = False controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS GET POST HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) self.assertEquals(len(resp.headers['Allow'].split(', ')), 4) # Test a CORS OPTIONS request (i.e. including Origin and # Access-Control-Request-Method headers) self.app.allow_account_management = False controller = proxy_server.AccountController(self.app, 'account') req = Request.blank( '/v1/account', {'REQUEST_METHOD': 'OPTIONS'}, headers={'Origin': 'http://foo.com', 'Access-Control-Request-Method': 'GET'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS GET POST HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) self.assertEquals(len(resp.headers['Allow'].split(', ')), 4) self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'}) req.content_length = 0 resp = controller.OPTIONS(req) self.assertEquals(200, resp.status_int) for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split(): self.assertTrue( verb in resp.headers['Allow']) self.assertEquals(len(resp.headers['Allow'].split(', ')), 6) def test_GET(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') # GET returns after the first successful call to an Account Server self.assert_status_map(controller.GET, (200,), 200, 200) self.assert_status_map(controller.GET, (503, 200), 200, 200) self.assert_status_map(controller.GET, (503, 503, 200), 200, 200) self.assert_status_map(controller.GET, (204,), 204, 204) self.assert_status_map(controller.GET, (503, 204), 204, 204) self.assert_status_map(controller.GET, (503, 503, 204), 204, 204) self.assert_status_map(controller.GET, (404, 200), 200, 200) self.assert_status_map(controller.GET, (404, 404, 200), 200, 200) self.assert_status_map(controller.GET, (404, 503, 204), 204, 204) # If Account servers fail, if autocreate = False, return majority # response self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) self.assert_status_map(controller.GET, (404, 404, 503), 404, 404) self.assert_status_map(controller.GET, (404, 503, 503), 503) self.app.memcache = FakeMemcacheReturnsNone() self.assert_status_map(controller.GET, (404, 404, 404), 404, 404) def test_GET_autocreate(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) # Repeat the test for autocreate = False and 404 by all self.assert_status_map(controller.GET, (404, 404, 404), 404) self.assert_status_map(controller.GET, (404, 503, 404), 404) # When autocreate is True, if none of the nodes respond 2xx # And quorum of the nodes responded 404, # ALL nodes are asked to create the account # If successful, the GET request is repeated. controller.app.account_autocreate = True self.assert_status_map(controller.GET, (404, 404, 404), 204) self.assert_status_map(controller.GET, (404, 503, 404), 204) # We always return 503 if no majority between 4xx, 3xx or 2xx found self.assert_status_map(controller.GET, (500, 500, 400), 503) def test_HEAD(self): # Same behaviour as GET with save_globals(): controller = proxy_server.AccountController(self.app, 'account') self.assert_status_map(controller.HEAD, (200,), 200, 200) self.assert_status_map(controller.HEAD, (503, 200), 200, 200) self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200) self.assert_status_map(controller.HEAD, (204,), 204, 204) self.assert_status_map(controller.HEAD, (503, 204), 204, 204) self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204) self.assert_status_map(controller.HEAD, (204,), 204, 204) self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404) self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200) self.assert_status_map(controller.HEAD, (404, 200), 200, 200) self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404) self.assert_status_map(controller.HEAD, (404, 503, 503), 503) self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204) def test_HEAD_autocreate(self): # Same behaviour as GET with save_globals(): controller = proxy_server.AccountController(self.app, 'account') self.app.memcache = FakeMemcacheReturnsNone() self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.HEAD, (404, 404, 404), 404) controller.app.account_autocreate = True self.assert_status_map(controller.HEAD, (404, 404, 404), 204) self.assert_status_map(controller.HEAD, (500, 404, 404), 204) # We always return 503 if no majority between 4xx, 3xx or 2xx found self.assert_status_map(controller.HEAD, (500, 500, 400), 503) def test_POST_autocreate(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') self.app.memcache = FakeMemcacheReturnsNone() # first test with autocreate being False self.assertFalse(self.app.account_autocreate) self.assert_status_map(controller.POST, (404, 404, 404), 404) # next turn it on and test account being created than updated controller.app.account_autocreate = True self.assert_status_map( controller.POST, (404, 404, 404, 202, 202, 202, 201, 201, 201), 201) # account_info PUT account POST account self.assert_status_map( controller.POST, (404, 404, 503, 201, 201, 503, 204, 204, 504), 204) # what if create fails self.assert_status_map( controller.POST, (404, 404, 404, 403, 403, 403, 400, 400, 400), 400) def test_connection_refused(self): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = 1 # can't connect on this port controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) def test_other_socket_error(self): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): dev['ip'] = '127.0.0.1' dev['port'] = -1 # invalid port number controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'}) self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) def test_response_get_accept_ranges_header(self): with save_globals(): set_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/a?format=json') self.app.update_request(req) res = controller.GET(req) self.assert_('accept-ranges' in res.headers) self.assertEqual(res.headers['accept-ranges'], 'bytes') def test_response_head_accept_ranges_header(self): with save_globals(): set_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/v1/a?format=json') self.app.update_request(req) res = controller.HEAD(req) res.body self.assert_('accept-ranges' in res.headers) self.assertEqual(res.headers['accept-ranges'], 'bytes') def test_PUT(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a', {}) req.content_length = 0 self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((201, 201, 201), 405) self.app.allow_account_management = True test_status_map((201, 201, 201), 201) test_status_map((201, 201, 500), 201) test_status_map((201, 500, 500), 503) test_status_map((204, 500, 404), 503) def test_PUT_max_account_name_length(self): with save_globals(): self.app.allow_account_management = True limit = MAX_ACCOUNT_NAME_LENGTH controller = proxy_server.AccountController(self.app, '1' * limit) self.assert_status_map(controller.PUT, (201, 201, 201), 201) controller = proxy_server.AccountController( self.app, '2' * (limit + 1)) self.assert_status_map(controller.PUT, (201, 201, 201), 400) def test_PUT_connect_exceptions(self): with save_globals(): self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'account') self.assert_status_map(controller.PUT, (201, 201, -1), 201) self.assert_status_map(controller.PUT, (201, -1, -1), 503) self.assert_status_map(controller.PUT, (503, 503, -1), 503) def test_PUT_metadata(self): self.metadata_helper('PUT') def test_POST_metadata(self): self.metadata_helper('POST') def metadata_helper(self, method): for test_header, test_value in ( ('X-Account-Meta-TestHeader', 'TestValue'), ('X-Account-Meta-TestHeader', ''), ('X-Remove-Account-Meta-TestHeader', 'anything')): test_errors = [] def test_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None): if path == '/a': find_header = test_header find_value = test_value if find_header.lower().startswith('x-remove-'): find_header = \ find_header.lower().replace('-remove', '', 1) find_value = '' for k, v in headers.iteritems(): if k.lower() == find_header.lower() and \ v == find_value: break else: test_errors.append('%s: %s not in %s' % (find_header, find_value, headers)) with save_globals(): self.app.allow_account_management = True controller = \ proxy_server.AccountController(self.app, 'a') set_http_connect(201, 201, 201, give_connect=test_connect) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={test_header: test_value}) self.app.update_request(req) getattr(controller, method)(req) self.assertEquals(test_errors, []) def test_PUT_bad_metadata(self): self.bad_metadata_helper('PUT') def test_POST_bad_metadata(self): self.bad_metadata_helper('POST') def bad_metadata_helper(self, method): with save_globals(): self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'a') set_http_connect(200, 201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Account-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) set_http_connect(201, 201, 201) headers = {} header_value = 'a' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Account-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Account-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) set_http_connect(201, 201, 201) headers['X-Account-Meta-a'] = \ 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method}, headers=headers) self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) def test_DELETE(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'}) req.content_length = 0 self.app.update_request(req) res = controller.DELETE(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((201, 201, 201), 405) self.app.allow_account_management = True test_status_map((201, 201, 201), 201) test_status_map((201, 201, 500), 201) test_status_map((201, 500, 500), 503) test_status_map((204, 500, 404), 503) def test_DELETE_with_query_string(self): # Extra safety in case someone typos a query string for an # account-level DELETE request that was really meant to be caught by # some middleware. with save_globals(): controller = proxy_server.AccountController(self.app, 'account') def test_status_map(statuses, expected, **kwargs): set_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/v1/a?whoops', environ={'REQUEST_METHOD': 'DELETE'}) req.content_length = 0 self.app.update_request(req) res = controller.DELETE(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) test_status_map((201, 201, 201), 400) self.app.allow_account_management = True test_status_map((201, 201, 201), 400) test_status_map((201, 201, 500), 400) test_status_map((201, 500, 500), 400) test_status_map((204, 500, 404), 400) class TestAccountControllerFakeGetResponse(unittest.TestCase): """ Test all the faked-out GET responses for accounts that don't exist. They have to match the responses for empty accounts that really exist. """ def setUp(self): conf = {'account_autocreate': 'yes'} self.app = proxy_server.Application(conf, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing) self.app.memcache = FakeMemcacheReturnsNone() def test_GET_autocreate_accept_json(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank( '/v1/a', headers={'Accept': 'application/json'}, environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a'}) resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/json; charset=utf-8', resp.headers['Content-Type']) self.assertEqual("[]", resp.body) def test_GET_autocreate_format_json(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank('/v1/a?format=json', environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a', 'QUERY_STRING': 'format=json'}) resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/json; charset=utf-8', resp.headers['Content-Type']) self.assertEqual("[]", resp.body) def test_GET_autocreate_accept_xml(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank('/v1/a', headers={"Accept": "text/xml"}, environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a'}) resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('text/xml; charset=utf-8', resp.headers['Content-Type']) empty_xml_listing = ('\n' '\n') self.assertEqual(empty_xml_listing, resp.body) def test_GET_autocreate_format_xml(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank('/v1/a?format=xml', environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a', 'QUERY_STRING': 'format=xml'}) resp = req.get_response(self.app) self.assertEqual(200, resp.status_int) self.assertEqual('application/xml; charset=utf-8', resp.headers['Content-Type']) empty_xml_listing = ('\n' '\n') self.assertEqual(empty_xml_listing, resp.body) def test_GET_autocreate_accept_unknown(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank('/v1/a', headers={"Accept": "mystery/meat"}, environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a'}) resp = req.get_response(self.app) self.assertEqual(406, resp.status_int) def test_GET_autocreate_format_invalid_utf8(self): with save_globals(): set_http_connect(*([404] * 100)) # nonexistent: all backends 404 req = Request.blank('/v1/a?format=\xff\xfe', environ={'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/a', 'QUERY_STRING': 'format=\xff\xfe'}) resp = req.get_response(self.app) self.assertEqual(400, resp.status_int) def test_account_acl_header_access(self): acl = { 'admin': ['AUTH_alice'], 'read-write': ['AUTH_bob'], 'read-only': ['AUTH_carol'], } prefix = get_sys_meta_prefix('account') privileged_headers = {(prefix + 'core-access-control'): format_acl( version=2, acl_dict=acl)} app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) with save_globals(): # Mock account server will provide privileged information (ACLs) set_http_connect(200, 200, 200, headers=privileged_headers) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET'}) resp = app.handle_request(req) # Not a swift_owner -- ACLs should NOT be in response header = 'X-Account-Access-Control' self.assert_(header not in resp.headers, '%r was in %r' % ( header, resp.headers)) # Same setup -- mock acct server will provide ACLs set_http_connect(200, 200, 200, headers=privileged_headers) req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET', 'swift_owner': True}) resp = app.handle_request(req) # For a swift_owner, the ACLs *should* be in response self.assert_(header in resp.headers, '%r not in %r' % ( header, resp.headers)) def test_account_acls_through_delegation(self): # Define a way to grab the requests sent out from the AccountController # to the Account Server, and a way to inject responses we'd like the # Account Server to return. resps_to_send = [] @contextmanager def patch_account_controller_method(verb): old_method = getattr(proxy_server.AccountController, verb) new_method = lambda self, req, *_, **__: resps_to_send.pop(0) try: setattr(proxy_server.AccountController, verb, new_method) yield finally: setattr(proxy_server.AccountController, verb, old_method) def make_test_request(http_method, swift_owner=True): env = { 'REQUEST_METHOD': http_method, 'swift_owner': swift_owner, } acl = { 'admin': ['foo'], 'read-write': ['bar'], 'read-only': ['bas'], } headers = {} if http_method in ('GET', 'HEAD') else { 'x-account-access-control': format_acl(version=2, acl_dict=acl) } return Request.blank('/v1/a', environ=env, headers=headers) # Our AccountController will invoke methods to communicate with the # Account Server, and they will return responses like these: def make_canned_response(http_method): acl = { 'admin': ['foo'], 'read-write': ['bar'], 'read-only': ['bas'], } headers = {'x-account-sysmeta-core-access-control': format_acl( version=2, acl_dict=acl)} canned_resp = Response(headers=headers) canned_resp.environ = { 'PATH_INFO': '/acct', 'REQUEST_METHOD': http_method, } resps_to_send.append(canned_resp) app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) app.allow_account_management = True ext_header = 'x-account-access-control' with patch_account_controller_method('GETorHEAD_base'): # GET/HEAD requests should remap sysmeta headers from acct server for verb in ('GET', 'HEAD'): make_canned_response(verb) req = make_test_request(verb) resp = app.handle_request(req) h = parse_acl(version=2, data=resp.headers.get(ext_header)) self.assertEqual(h['admin'], ['foo']) self.assertEqual(h['read-write'], ['bar']) self.assertEqual(h['read-only'], ['bas']) # swift_owner = False: GET/HEAD shouldn't return sensitive info make_canned_response(verb) req = make_test_request(verb, swift_owner=False) resp = app.handle_request(req) h = resp.headers self.assertEqual(None, h.get(ext_header)) # swift_owner unset: GET/HEAD shouldn't return sensitive info make_canned_response(verb) req = make_test_request(verb, swift_owner=False) del req.environ['swift_owner'] resp = app.handle_request(req) h = resp.headers self.assertEqual(None, h.get(ext_header)) # Verify that PUT/POST requests remap sysmeta headers from acct server with patch_account_controller_method('make_requests'): make_canned_response('PUT') req = make_test_request('PUT') resp = app.handle_request(req) h = parse_acl(version=2, data=resp.headers.get(ext_header)) self.assertEqual(h['admin'], ['foo']) self.assertEqual(h['read-write'], ['bar']) self.assertEqual(h['read-only'], ['bas']) make_canned_response('POST') req = make_test_request('POST') resp = app.handle_request(req) h = parse_acl(version=2, data=resp.headers.get(ext_header)) self.assertEqual(h['admin'], ['foo']) self.assertEqual(h['read-write'], ['bar']) self.assertEqual(h['read-only'], ['bas']) class FakeObjectController(object): def __init__(self): self.app = self self.logger = self self.account_name = 'a' self.container_name = 'c' self.object_name = 'o' self.trans_id = 'tx1' self.object_ring = FakeRing() self.node_timeout = 1 self.rate_limit_after_segment = 3 self.rate_limit_segments_per_sec = 2 self.GETorHEAD_base_args = [] def exception(self, *args): self.exception_args = args self.exception_info = sys.exc_info() def GETorHEAD_base(self, *args): self.GETorHEAD_base_args.append(args) req = args[0] path = args[4] body = data = path[-1] * int(path[-1]) if req.range: r = req.range.ranges_for_length(len(data)) if r: (start, stop) = r[0] body = data[start:stop] resp = Response(app_iter=iter(body)) return resp def iter_nodes(self, ring, partition): for node in ring.get_part_nodes(partition): yield node for node in ring.get_more_nodes(partition): yield node def sort_nodes(self, nodes): return nodes def set_node_timing(self, node, timing): return class Stub(object): pass class TestProxyObjectPerformance(unittest.TestCase): def setUp(self): # This is just a simple test that can be used to verify and debug the # various data paths between the proxy server and the object # server. Used as a play ground to debug buffer sizes for sockets. prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) # Client is transmitting in 2 MB chunks fd = sock.makefile('wb', 2 * 1024 * 1024) # Small, fast for testing obj_len = 2 * 64 * 1024 # Use 1 GB or more for measurements #obj_len = 2 * 512 * 1024 * 1024 self.path = '/v1/a/c/o.large' fd.write('PUT %s HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' 'Content-Length: %s\r\n' 'Content-Type: application/octet-stream\r\n' '\r\n' % (self.path, str(obj_len))) fd.write('a' * obj_len) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEqual(headers[:len(exp)], exp) self.obj_len = obj_len def test_GET_debug_large_file(self): for i in range(10): start = time.time() prolis = _test_sockets[0] sock = connect_tcp(('localhost', prolis.getsockname()[1])) # Client is reading in 2 MB chunks fd = sock.makefile('wb', 2 * 1024 * 1024) fd.write('GET %s HTTP/1.1\r\n' 'Host: localhost\r\n' 'Connection: close\r\n' 'X-Storage-Token: t\r\n' '\r\n' % self.path) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEqual(headers[:len(exp)], exp) total = 0 while True: buf = fd.read(100000) if not buf: break total += len(buf) self.assertEqual(total, self.obj_len) end = time.time() print "Run %02d took %07.03f" % (i, end - start) class TestSwiftInfo(unittest.TestCase): def setUp(self): utils._swift_info = {} utils._swift_admin_info = {} def test_registered_defaults(self): proxy_server.Application({}, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing) si = utils.get_swift_info()['swift'] self.assertTrue('version' in si) self.assertEqual(si['max_file_size'], MAX_FILE_SIZE) self.assertEqual(si['max_meta_name_length'], MAX_META_NAME_LENGTH) self.assertEqual(si['max_meta_value_length'], MAX_META_VALUE_LENGTH) self.assertEqual(si['max_meta_count'], MAX_META_COUNT) self.assertEqual(si['account_listing_limit'], ACCOUNT_LISTING_LIMIT) self.assertEqual(si['container_listing_limit'], CONTAINER_LISTING_LIMIT) self.assertEqual(si['max_account_name_length'], MAX_ACCOUNT_NAME_LENGTH) self.assertEqual(si['max_container_name_length'], MAX_CONTAINER_NAME_LENGTH) self.assertEqual(si['max_object_name_length'], MAX_OBJECT_NAME_LENGTH) if __name__ == '__main__': setup() try: unittest.main() finally: teardown() swift-1.13.1/test/unit/proxy/controllers/0000775000175400017540000000000012323703665021563 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/proxy/controllers/test_container.py0000664000175400017540000001300712323703611025146 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mock import unittest from swift.common.swob import Request from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_container_info from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.common.request_helpers import get_sys_meta_prefix class TestContainerController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) def test_container_info_in_response_env(self): controller = proxy_server.ContainerController(self.app, 'a', 'c') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, body='')): req = Request.blank('/v1/a/c', {'PATH_INFO': '/v1/a/c'}) resp = controller.HEAD(req) self.assertEqual(2, resp.status_int // 100) self.assertTrue("swift.container/a/c" in resp.environ) self.assertEqual(headers_to_container_info(resp.headers), resp.environ['swift.container/a/c']) def test_swift_owner(self): owner_headers = { 'x-container-read': 'value', 'x-container-write': 'value', 'x-container-sync-key': 'value', 'x-container-sync-to': 'value'} controller = proxy_server.ContainerController(self.app, 'a', 'c') req = Request.blank('/v1/a/c') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, headers=owner_headers)): resp = controller.HEAD(req) self.assertEquals(2, resp.status_int // 100) for key in owner_headers: self.assertTrue(key not in resp.headers) req = Request.blank('/v1/a/c', environ={'swift_owner': True}) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, headers=owner_headers)): resp = controller.HEAD(req) self.assertEquals(2, resp.status_int // 100) for key in owner_headers: self.assertTrue(key in resp.headers) def _make_callback_func(self, context): def callback(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): context['method'] = method context['path'] = path context['headers'] = headers or {} return callback def test_sys_meta_headers_PUT(self): # check that headers in sys meta namespace make it through # the container controller sys_meta_key = '%stest' % get_sys_meta_prefix('container') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Container-Meta-Test' controller = proxy_server.ContainerController(self.app, 'a', 'c') context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a/c', headers=hdrs_in) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, give_connect=callback)): controller.PUT(req) self.assertEqual(context['method'], 'PUT') self.assertTrue(sys_meta_key in context['headers']) self.assertEqual(context['headers'][sys_meta_key], 'foo') self.assertTrue(user_meta_key in context['headers']) self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0') def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the container controller sys_meta_key = '%stest' % get_sys_meta_prefix('container') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Container-Meta-Test' controller = proxy_server.ContainerController(self.app, 'a', 'c') context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a/c', headers=hdrs_in) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, give_connect=callback)): controller.POST(req) self.assertEqual(context['method'], 'POST') self.assertTrue(sys_meta_key in context['headers']) self.assertEqual(context['headers'][sys_meta_key], 'foo') self.assertTrue(user_meta_key in context['headers']) self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/proxy/controllers/test_info.py0000664000175400017540000003036112323703611024121 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import time from mock import Mock from swift.proxy.controllers import InfoController from swift.proxy.server import Application as ProxyApp from swift.common import utils from swift.common.utils import json from swift.common.swob import Request, HTTPException class TestInfoController(unittest.TestCase): def setUp(self): utils._swift_info = {} utils._swift_admin_info = {} def get_controller(self, expose_info=None, disallowed_sections=None, admin_key=None): disallowed_sections = disallowed_sections or [] app = Mock(spec=ProxyApp) return InfoController(app, None, expose_info, disallowed_sections, admin_key) def start_response(self, status, headers): self.got_statuses.append(status) for h in headers: self.got_headers.append({h[0]: h[1]}) def test_disabled_info(self): controller = self.get_controller(expose_info=False) req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('403 Forbidden', str(resp)) def test_get_info(self): controller = self.get_controller(expose_info=True) utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) info = json.loads(resp.body) self.assertTrue('admin' not in info) self.assertTrue('foo' in info) self.assertTrue('bar' in info['foo']) self.assertEqual(info['foo']['bar'], 'baz') def test_options_info(self): controller = self.get_controller(expose_info=True) req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) resp = controller.OPTIONS(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) self.assertTrue('Allow' in resp.headers) def test_get_info_cors(self): controller = self.get_controller(expose_info=True) utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}, headers={'Origin': 'http://example.com'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) info = json.loads(resp.body) self.assertTrue('admin' not in info) self.assertTrue('foo' in info) self.assertTrue('bar' in info['foo']) self.assertEqual(info['foo']['bar'], 'baz') self.assertTrue('Access-Control-Allow-Origin' in resp.headers) self.assertTrue('Access-Control-Expose-Headers' in resp.headers) def test_head_info(self): controller = self.get_controller(expose_info=True) utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'HEAD'}) resp = controller.HEAD(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) def test_disallow_info(self): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2']) utils._swift_info = {'foo': {'bar': 'baz'}, 'foo2': {'bar2': 'baz2'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) info = json.loads(resp.body) self.assertTrue('foo' in info) self.assertTrue('bar' in info['foo']) self.assertEqual(info['foo']['bar'], 'baz') self.assertTrue('foo2' not in info) def test_disabled_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, '') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('403 Forbidden', str(resp)) def test_get_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) info = json.loads(resp.body) self.assertTrue('admin' in info) self.assertTrue('qux' in info['admin']) self.assertTrue('quux' in info['admin']['qux']) self.assertEqual(info['admin']['qux']['quux'], 'corge') def test_head_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'HEAD'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) expires = int(time.time() + 86400) sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'HEAD'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) def test_get_admin_info_invalid_method(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('401 Unauthorized', str(resp)) def test_get_admin_info_invalid_expires(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = 1 sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('401 Unauthorized', str(resp)) expires = 'abc' sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('401 Unauthorized', str(resp)) def test_get_admin_info_invalid_path(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('401 Unauthorized', str(resp)) def test_get_admin_info_invalid_key(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'invalid-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('401 Unauthorized', str(resp)) def test_admin_disallow_info(self): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2'], admin_key='secret-admin-key') utils._swift_info = {'foo': {'bar': 'baz'}, 'foo2': {'bar2': 'baz2'}} utils._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') path = '/info?swiftinfo_sig={sig}&swiftinfo_expires={expires}'.format( sig=sig, expires=expires) req = Request.blank( path, environ={'REQUEST_METHOD': 'GET'}) resp = controller.GET(req) self.assertTrue(isinstance(resp, HTTPException)) self.assertEqual('200 OK', str(resp)) info = json.loads(resp.body) self.assertTrue('foo2' not in info) self.assertTrue('admin' in info) self.assertTrue('disallowed_sections' in info['admin']) self.assertTrue('foo2' in info['admin']['disallowed_sections']) self.assertTrue('qux' in info['admin']) self.assertTrue('quux' in info['admin']['qux']) self.assertEqual(info['admin']['qux']['quux'], 'corge') if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/proxy/controllers/test_obj.py0000775000175400017540000001765412323703614023760 0ustar jenkinsjenkins00000000000000#!/usr/bin/env python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from contextlib import contextmanager import mock import swift from swift.proxy import server as proxy_server from swift.common.swob import HTTPException from test.unit import FakeRing, FakeMemcache, fake_http_connect, debug_logger @contextmanager def set_http_connect(*args, **kwargs): old_connect = swift.proxy.controllers.base.http_connect new_connect = fake_http_connect(*args, **kwargs) swift.proxy.controllers.base.http_connect = new_connect swift.proxy.controllers.obj.http_connect = new_connect swift.proxy.controllers.account.http_connect = new_connect swift.proxy.controllers.container.http_connect = new_connect yield new_connect swift.proxy.controllers.base.http_connect = old_connect swift.proxy.controllers.obj.http_connect = old_connect swift.proxy.controllers.account.http_connect = old_connect swift.proxy.controllers.container.http_connect = old_connect class TestObjControllerWriteAffinity(unittest.TestCase): def setUp(self): self.app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9)) self.app.request_node_count = lambda replicas: 10000000 self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes def test_iter_nodes_local_first_noops_when_no_affinity(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') self.app.write_affinity_is_local_fn = None all_nodes = self.app.object_ring.get_part_nodes(1) all_nodes.extend(self.app.object_ring.get_more_nodes(1)) local_first_nodes = list(controller.iter_nodes_local_first( self.app.object_ring, 1)) self.maxDiff = None self.assertEqual(all_nodes, local_first_nodes) def test_iter_nodes_local_first_moves_locals_first(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') self.app.write_affinity_is_local_fn = ( lambda node: node['region'] == 1) self.app.write_affinity_node_count = lambda ring: 4 all_nodes = self.app.object_ring.get_part_nodes(1) all_nodes.extend(self.app.object_ring.get_more_nodes(1)) local_first_nodes = list(controller.iter_nodes_local_first( self.app.object_ring, 1)) # the local nodes move up in the ordering self.assertEqual([1, 1, 1, 1], [node['region'] for node in local_first_nodes[:4]]) # we don't skip any nodes self.assertEqual(sorted(all_nodes), sorted(local_first_nodes)) def test_connect_put_node_timeout(self): controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') self.app.conn_timeout = 0.1 with set_http_connect(200, slow_connect=True): nodes = [dict(ip='', port='', device='')] res = controller._connect_put_node(nodes, '', '', {}, ('', '')) self.assertTrue(res is None) class TestObjController(unittest.TestCase): def setUp(self): logger = debug_logger('proxy-server') logger.thread_locals = ('txn1', '127.0.0.2') self.app = proxy_server.Application( None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing(), logger=logger) self.controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') self.controller.container_info = mock.MagicMock(return_value={ 'partition': 1, 'nodes': [ {'ip': '127.0.0.1', 'port': '1', 'device': 'sda'}, {'ip': '127.0.0.1', 'port': '2', 'device': 'sda'}, {'ip': '127.0.0.1', 'port': '3', 'device': 'sda'}, ], 'write_acl': None, 'read_acl': None, 'sync_key': None, 'versions': None}) def test_PUT_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['content-length'] = '0' with set_http_connect(201, 201, 201): resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 201) def test_PUT_if_none_match(self): req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['if-none-match'] = '*' req.headers['content-length'] = '0' with set_http_connect(201, 201, 201): resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 201) def test_PUT_if_none_match_denied(self): req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['if-none-match'] = '*' req.headers['content-length'] = '0' with set_http_connect(201, (412, 412), 201): resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 412) def test_PUT_if_none_match_not_star(self): req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['if-none-match'] = 'somethingelse' req.headers['content-length'] = '0' with set_http_connect(201, 201, 201): resp = self.controller.PUT(req) self.assertEquals(resp.status_int, 400) def test_GET_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') with set_http_connect(200): resp = self.controller.GET(req) self.assertEquals(resp.status_int, 200) def test_DELETE_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') with set_http_connect(204, 204, 204): resp = self.controller.DELETE(req) self.assertEquals(resp.status_int, 204) def test_POST_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') with set_http_connect(200, 200, 200, 201, 201, 201): resp = self.controller.POST(req) self.assertEquals(resp.status_int, 202) def test_COPY_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') with set_http_connect(200, 200, 200, 201, 201, 201): resp = self.controller.POST(req) self.assertEquals(resp.status_int, 202) def test_HEAD_simple(self): req = swift.common.swob.Request.blank('/v1/a/c/o') with set_http_connect(200, 200, 200, 201, 201, 201): resp = self.controller.POST(req) self.assertEquals(resp.status_int, 202) def test_PUT_log_info(self): # mock out enough to get to the area of the code we want to test with mock.patch('swift.proxy.controllers.obj.check_object_creation', mock.MagicMock(return_value=None)): req = swift.common.swob.Request.blank('/v1/a/c/o') req.headers['x-copy-from'] = 'somewhere' try: self.controller.PUT(req) except HTTPException: pass self.assertEquals( req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) # and then check that we don't do that for originating POSTs req = swift.common.swob.Request.blank('/v1/a/c/o') req.method = 'POST' req.headers['x-copy-from'] = 'elsewhere' try: self.controller.PUT(req) except HTTPException: pass self.assertEquals(req.environ.get('swift.log_info'), None) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/proxy/controllers/test_account.py0000664000175400017540000002535612323703614024635 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mock import unittest from swift.common.swob import Request, Response from swift.common.middleware.acl import format_acl from swift.proxy import server as proxy_server from swift.proxy.controllers.base import headers_to_account_info from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.common.request_helpers import get_sys_meta_prefix import swift.proxy.controllers.base class TestAccountController(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) def test_account_info_in_response_env(self): controller = proxy_server.AccountController(self.app, 'AUTH_bob') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, body='')): req = Request.blank('/v1/AUTH_bob', {'PATH_INFO': '/v1/AUTH_bob'}) resp = controller.HEAD(req) self.assertEqual(2, resp.status_int // 100) self.assertTrue('swift.account/AUTH_bob' in resp.environ) self.assertEqual(headers_to_account_info(resp.headers), resp.environ['swift.account/AUTH_bob']) def test_swift_owner(self): owner_headers = { 'x-account-meta-temp-url-key': 'value', 'x-account-meta-temp-url-key-2': 'value'} controller = proxy_server.AccountController(self.app, 'a') req = Request.blank('/v1/a') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, headers=owner_headers)): resp = controller.HEAD(req) self.assertEquals(2, resp.status_int // 100) for key in owner_headers: self.assertTrue(key not in resp.headers) req = Request.blank('/v1/a', environ={'swift_owner': True}) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, headers=owner_headers)): resp = controller.HEAD(req) self.assertEquals(2, resp.status_int // 100) for key in owner_headers: self.assertTrue(key in resp.headers) def test_get_deleted_account(self): resp_headers = { 'x-account-status': 'deleted', } controller = proxy_server.AccountController(self.app, 'a') req = Request.blank('/v1/a') with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(404, headers=resp_headers)): resp = controller.HEAD(req) self.assertEquals(410, resp.status_int) def test_long_acct_names(self): long_acct_name = '%sLongAccountName' % ('Very' * (MAX_ANAME_LEN // 4)) controller = proxy_server.AccountController(self.app, long_acct_name) req = Request.blank('/v1/%s' % long_acct_name) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200)): resp = controller.HEAD(req) self.assertEquals(400, resp.status_int) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200)): resp = controller.GET(req) self.assertEquals(400, resp.status_int) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200)): resp = controller.POST(req) self.assertEquals(400, resp.status_int) def _make_callback_func(self, context): def callback(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): context['method'] = method context['path'] = path context['headers'] = headers or {} return callback def test_sys_meta_headers_PUT(self): # check that headers in sys meta namespace make it through # the proxy controller sys_meta_key = '%stest' % get_sys_meta_prefix('account') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Account-Meta-Test' # allow PUTs to account... self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'a') context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a', headers=hdrs_in) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, give_connect=callback)): controller.PUT(req) self.assertEqual(context['method'], 'PUT') self.assertTrue(sys_meta_key in context['headers']) self.assertEqual(context['headers'][sys_meta_key], 'foo') self.assertTrue(user_meta_key in context['headers']) self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0') def test_sys_meta_headers_POST(self): # check that headers in sys meta namespace make it through # the proxy controller sys_meta_key = '%stest' % get_sys_meta_prefix('account') sys_meta_key = sys_meta_key.title() user_meta_key = 'X-Account-Meta-Test' controller = proxy_server.AccountController(self.app, 'a') context = {} callback = self._make_callback_func(context) hdrs_in = {sys_meta_key: 'foo', user_meta_key: 'bar', 'x-timestamp': '1.0'} req = Request.blank('/v1/a', headers=hdrs_in) with mock.patch('swift.proxy.controllers.base.http_connect', fake_http_connect(200, 200, give_connect=callback)): controller.POST(req) self.assertEqual(context['method'], 'POST') self.assertTrue(sys_meta_key in context['headers']) self.assertEqual(context['headers'][sys_meta_key], 'foo') self.assertTrue(user_meta_key in context['headers']) self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertNotEqual(context['headers']['x-timestamp'], '1.0') def _make_user_and_sys_acl_headers_data(self): acl = { 'admin': ['AUTH_alice', 'AUTH_bob'], 'read-write': ['AUTH_carol'], 'read-only': [], } user_prefix = 'x-account-' # external, user-facing user_headers = {(user_prefix + 'access-control'): format_acl( version=2, acl_dict=acl)} sys_prefix = get_sys_meta_prefix('account') # internal, system-facing sys_headers = {(sys_prefix + 'core-access-control'): format_acl( version=2, acl_dict=acl)} return user_headers, sys_headers def test_account_acl_headers_translated_for_GET_HEAD(self): # Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers # from the account server will remap those headers to X-Account-Acl-* hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() controller = proxy_server.AccountController(self.app, 'acct') for verb in ('GET', 'HEAD'): req = Request.blank('/v1/acct', environ={'swift_owner': True}) controller.GETorHEAD_base = lambda *_: Response( headers=hdrs_int, environ={ 'PATH_INFO': '/acct', 'REQUEST_METHOD': verb, }) method = getattr(controller, verb) resp = method(req) for header, value in hdrs_ext.items(): if value: self.assertEqual(resp.headers.get(header), value) else: # blank ACLs should result in no header self.assert_(header not in resp.headers) def test_add_acls_impossible_cases(self): # For test coverage: verify that defensive coding does defend, in cases # that shouldn't arise naturally # add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST resp = Response() controller = proxy_server.AccountController(self.app, 'a') resp.environ['PATH_INFO'] = '/a' resp.environ['REQUEST_METHOD'] = 'OPTIONS' controller.add_acls_from_sys_metadata(resp) self.assertEqual(1, len(resp.headers)) # we always get Content-Type self.assertEqual(2, len(resp.environ)) def test_memcache_key_impossible_cases(self): # For test coverage: verify that defensive coding does defend, in cases # that shouldn't arise naturally self.assertRaises( ValueError, lambda: swift.proxy.controllers.base.get_container_memcache_key( '/a', None)) def test_stripping_swift_admin_headers(self): # Verify that a GET/HEAD which receives privileged headers from the # account server will strip those headers for non-swift_owners hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data() headers = { 'x-account-meta-harmless': 'hi mom', 'x-account-meta-temp-url-key': 's3kr1t', } controller = proxy_server.AccountController(self.app, 'acct') for verb in ('GET', 'HEAD'): for env in ({'swift_owner': True}, {'swift_owner': False}): req = Request.blank('/v1/acct', environ=env) controller.GETorHEAD_base = lambda *_: Response( headers=headers, environ={ 'PATH_INFO': '/acct', 'REQUEST_METHOD': verb, }) method = getattr(controller, verb) resp = method(req) self.assertEqual(resp.headers.get('x-account-meta-harmless'), 'hi mom') privileged_header_present = ( 'x-account-meta-temp-url-key' in resp.headers) self.assertEqual(privileged_header_present, env['swift_owner']) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/proxy/controllers/test_base.py0000664000175400017540000005703112323703611024103 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ headers_to_account_info, headers_to_object_info, get_container_info, \ get_container_memcache_key, get_account_info, get_account_memcache_key, \ get_object_env_key, _get_cache_key, get_info, get_object_info, \ Controller, GetOrHeadHandler from swift.common.swob import Request, HTTPException, HeaderKeyDict from swift.common.utils import split_path from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server from swift.common.request_helpers import get_sys_meta_prefix FakeResponse_status_int = 201 class FakeResponse(object): def __init__(self, headers, env, account, container, obj): self.headers = headers self.status_int = FakeResponse_status_int self.environ = env if obj: env_key = get_object_env_key(account, container, obj) else: cache_key, env_key = _get_cache_key(account, container) if account and container and obj: info = headers_to_object_info(headers, FakeResponse_status_int) elif account and container: info = headers_to_container_info(headers, FakeResponse_status_int) else: info = headers_to_account_info(headers, FakeResponse_status_int) env[env_key] = info class FakeRequest(object): def __init__(self, env, path, swift_source=None): self.environ = env (version, account, container, obj) = split_path(path, 2, 4, True) self.account = account self.container = container self.obj = obj if obj: stype = 'object' self.headers = {'content-length': 5555, 'content-type': 'text/plain'} else: stype = container and 'container' or 'account' self.headers = {'x-%s-object-count' % (stype): 1000, 'x-%s-bytes-used' % (stype): 6666} if swift_source: meta = 'x-%s-meta-fakerequest-swift-source' % stype self.headers[meta] = swift_source def get_response(self, app): return FakeResponse(self.headers, self.environ, self.account, self.container, self.obj) class FakeCache(object): def __init__(self, val): self.val = val def get(self, *args): return self.val class TestFuncs(unittest.TestCase): def setUp(self): self.app = proxy_server.Application(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing) def test_GETorHEAD_base(self): base = Controller(self.app) req = Request.blank('/v1/a/c/o/with/slashes') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', '/a/c/o/with/slashes') self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ) self.assertEqual( resp.environ['swift.object/a/c/o/with/slashes']['status'], 200) req = Request.blank('/v1/a/c/o') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part', '/a/c/o') self.assertTrue('swift.object/a/c/o' in resp.environ) self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200) req = Request.blank('/v1/a/c') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'container', FakeRing(), 'part', '/a/c') self.assertTrue('swift.container/a/c' in resp.environ) self.assertEqual(resp.environ['swift.container/a/c']['status'], 200) req = Request.blank('/v1/a') with patch('swift.proxy.controllers.base.' 'http_connect', fake_http_connect(200)): resp = base.GETorHEAD_base(req, 'account', FakeRing(), 'part', '/a') self.assertTrue('swift.account/a' in resp.environ) self.assertEqual(resp.environ['swift.account/a']['status'], 200) def test_get_info(self): global FakeResponse_status_int # Do a non cached call to account env = {} with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): info_a = get_info(None, env, 'a') # Check that you got proper info self.assertEquals(info_a['status'], 201) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) # Do an env cached call to account info_a = get_info(None, env, 'a') # Check that you got proper info self.assertEquals(info_a['status'], 201) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) # This time do env cached call to account and non cached to container with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): info_c = get_info(None, env, 'a', 'c') # Check that you got proper info self.assertEquals(info_a['status'], 201) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do a non cached call to account than non cached to # container env = {} # abandon previous call to env with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): info_c = get_info(None, env, 'a', 'c') # Check that you got proper info self.assertEquals(info_a['status'], 201) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) self.assertEquals(env.get('swift.container/a/c'), info_c) # This time do an env cached call to container while account is not # cached del(env['swift.account/a']) info_c = get_info(None, env, 'a', 'c') # Check that you got proper info self.assertEquals(info_a['status'], 201) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set and account still not cached self.assertEquals(env.get('swift.container/a/c'), info_c) # Do a non cached call to account not found with ret_not_found env = {} with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): try: FakeResponse_status_int = 404 info_a = get_info(None, env, 'a', ret_not_found=True) finally: FakeResponse_status_int = 201 # Check that you got proper info self.assertEquals(info_a['status'], 404) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) # Do a cached call to account not found with ret_not_found info_a = get_info(None, env, 'a', ret_not_found=True) # Check that you got proper info self.assertEquals(info_a['status'], 404) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) # Do a non cached call to account not found without ret_not_found env = {} with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): try: FakeResponse_status_int = 404 info_a = get_info(None, env, 'a') finally: FakeResponse_status_int = 201 # Check that you got proper info self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) # Do a cached call to account not found without ret_not_found info_a = get_info(None, env, 'a') # Check that you got proper info self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) def test_get_container_info_swift_source(self): req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_container_info(req.environ, 'app', swift_source='MC') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') def test_get_object_info_swift_source(self): req = Request.blank("/v1/a/c/o", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'app', swift_source='LU') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU') def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['object_count'], 1000) def test_get_container_info_cache(self): cached = {'status': 404, 'bytes': 3333, 'object_count': 10, # simplejson sometimes hands back strings, sometimes unicodes 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404) self.assertEquals(resp['versions'], "\xe1\xbd\x8a\x39") def test_get_container_info_env(self): cache_key = get_container_memcache_key("account", "cont") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account/cont", environ={env_key: {'bytes': 3867}, 'swift.cache': FakeCache({})}) resp = get_container_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) def test_get_account_info_swift_source(self): req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'a', swift_source='MC') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') def test_get_account_info_no_cache(self): req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['total_object_count'], 1000) def test_get_account_info_cache(self): # The original test that we prefer to preserve cached = {'status': 404, 'bytes': 3333, 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['total_object_count'], 10) self.assertEquals(resp['status'], 404) # Here is a more realistic test cached = {'status': 404, 'bytes': '3333', 'container_count': '234', 'total_object_count': '10', 'meta': {}} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['status'], 404) self.assertEquals(resp['bytes'], '3333') self.assertEquals(resp['container_count'], 234) self.assertEquals(resp['meta'], {}) self.assertEquals(resp['total_object_count'], '10') def test_get_account_info_env(self): cache_key = get_account_memcache_key("account") env_key = 'swift.%s' % cache_key req = Request.blank("/v1/account", environ={env_key: {'bytes': 3867}, 'swift.cache': FakeCache({})}) resp = get_account_info(req.environ, 'xxx') self.assertEquals(resp['bytes'], 3867) def test_get_object_info_env(self): cached = {'status': 200, 'length': 3333, 'type': 'application/json', 'meta': {}} env_key = get_object_env_key("account", "cont", "obj") req = Request.blank("/v1/account/cont/obj", environ={env_key: cached, 'swift.cache': FakeCache({})}) resp = get_object_info(req.environ, 'xxx') self.assertEquals(resp['length'], 3333) self.assertEquals(resp['type'], 'application/json') def test_get_object_info_no_env(self): req = Request.blank("/v1/account/cont/obj", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'xxx') self.assertEquals(resp['length'], 5555) self.assertEquals(resp['type'], 'text/plain') def test_headers_to_container_info_missing(self): resp = headers_to_container_info({}, 404) self.assertEquals(resp['status'], 404) self.assertEquals(resp['read_acl'], None) self.assertEquals(resp['write_acl'], None) def test_headers_to_container_info_meta(self): headers = {'X-Container-Meta-Whatevs': 14, 'x-container-meta-somethingelse': 0} resp = headers_to_container_info(headers.items(), 200) self.assertEquals(len(resp['meta']), 2) self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) def test_headers_to_container_info_sys_meta(self): prefix = get_sys_meta_prefix('container') headers = {'%sWhatevs' % prefix: 14, '%ssomethingelse' % prefix: 0} resp = headers_to_container_info(headers.items(), 200) self.assertEquals(len(resp['sysmeta']), 2) self.assertEquals(resp['sysmeta']['whatevs'], 14) self.assertEquals(resp['sysmeta']['somethingelse'], 0) def test_headers_to_container_info_values(self): headers = { 'x-container-read': 'readvalue', 'x-container-write': 'writevalue', 'x-container-sync-key': 'keyvalue', 'x-container-meta-access-control-allow-origin': 'here', } resp = headers_to_container_info(headers.items(), 200) self.assertEquals(resp['read_acl'], 'readvalue') self.assertEquals(resp['write_acl'], 'writevalue') self.assertEquals(resp['cors']['allow_origin'], 'here') headers['x-unused-header'] = 'blahblahblah' self.assertEquals( resp, headers_to_container_info(headers.items(), 200)) def test_headers_to_account_info_missing(self): resp = headers_to_account_info({}, 404) self.assertEquals(resp['status'], 404) self.assertEquals(resp['bytes'], None) self.assertEquals(resp['container_count'], None) def test_headers_to_account_info_meta(self): headers = {'X-Account-Meta-Whatevs': 14, 'x-account-meta-somethingelse': 0} resp = headers_to_account_info(headers.items(), 200) self.assertEquals(len(resp['meta']), 2) self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) def test_headers_to_account_info_sys_meta(self): prefix = get_sys_meta_prefix('account') headers = {'%sWhatevs' % prefix: 14, '%ssomethingelse' % prefix: 0} resp = headers_to_account_info(headers.items(), 200) self.assertEquals(len(resp['sysmeta']), 2) self.assertEquals(resp['sysmeta']['whatevs'], 14) self.assertEquals(resp['sysmeta']['somethingelse'], 0) def test_headers_to_account_info_values(self): headers = { 'x-account-object-count': '10', 'x-account-container-count': '20', } resp = headers_to_account_info(headers.items(), 200) self.assertEquals(resp['total_object_count'], '10') self.assertEquals(resp['container_count'], '20') headers['x-unused-header'] = 'blahblahblah' self.assertEquals( resp, headers_to_account_info(headers.items(), 200)) def test_headers_to_object_info_missing(self): resp = headers_to_object_info({}, 404) self.assertEquals(resp['status'], 404) self.assertEquals(resp['length'], None) self.assertEquals(resp['etag'], None) def test_headers_to_object_info_meta(self): headers = {'X-Object-Meta-Whatevs': 14, 'x-object-meta-somethingelse': 0} resp = headers_to_object_info(headers.items(), 200) self.assertEquals(len(resp['meta']), 2) self.assertEquals(resp['meta']['whatevs'], 14) self.assertEquals(resp['meta']['somethingelse'], 0) def test_headers_to_object_info_values(self): headers = { 'content-length': '1024', 'content-type': 'application/json', } resp = headers_to_object_info(headers.items(), 200) self.assertEquals(resp['length'], '1024') self.assertEquals(resp['type'], 'application/json') headers['x-unused-header'] = 'blahblahblah' self.assertEquals( resp, headers_to_object_info(headers.items(), 200)) def test_have_quorum(self): base = Controller(self.app) # just throw a bunch of test cases at it self.assertEqual(base.have_quorum([201, 404], 3), False) self.assertEqual(base.have_quorum([201, 201], 4), False) self.assertEqual(base.have_quorum([201, 201, 404, 404], 4), False) self.assertEqual(base.have_quorum([201, 503, 503, 201], 4), False) self.assertEqual(base.have_quorum([201, 201], 3), True) self.assertEqual(base.have_quorum([404, 404], 3), True) self.assertEqual(base.have_quorum([201, 201], 2), True) self.assertEqual(base.have_quorum([404, 404], 2), True) self.assertEqual(base.have_quorum([201, 404, 201, 201], 4), True) def test_range_fast_forward(self): req = Request.blank('/') handler = GetOrHeadHandler(None, req, None, None, None, None, {}) handler.fast_forward(50) self.assertEquals(handler.backend_headers['Range'], 'bytes=50-') handler = GetOrHeadHandler(None, req, None, None, None, None, {'Range': 'bytes=23-50'}) handler.fast_forward(20) self.assertEquals(handler.backend_headers['Range'], 'bytes=43-50') self.assertRaises(HTTPException, handler.fast_forward, 80) handler = GetOrHeadHandler(None, req, None, None, None, None, {'Range': 'bytes=23-'}) handler.fast_forward(20) self.assertEquals(handler.backend_headers['Range'], 'bytes=43-') handler = GetOrHeadHandler(None, req, None, None, None, None, {'Range': 'bytes=-100'}) handler.fast_forward(20) self.assertEquals(handler.backend_headers['Range'], 'bytes=-80') def test_transfer_headers_with_sysmeta(self): base = Controller(self.app) good_hdrs = {'x-base-sysmeta-foo': 'ok', 'X-Base-sysmeta-Bar': 'also ok'} bad_hdrs = {'x-base-sysmeta-': 'too short'} hdrs = dict(good_hdrs) hdrs.update(bad_hdrs) dst_hdrs = HeaderKeyDict() base.transfer_headers(hdrs, dst_hdrs) self.assertEqual(HeaderKeyDict(good_hdrs), dst_hdrs) def test_generate_request_headers(self): base = Controller(self.app) src_headers = {'x-remove-base-meta-owner': 'x', 'x-base-meta-size': '151M', 'new-owner': 'Kun'} req = Request.blank('/v1/a/c/o', headers=src_headers) dst_headers = base.generate_request_headers(req, transfer=True) expected_headers = {'x-base-meta-owner': '', 'x-base-meta-size': '151M'} for k, v in expected_headers.iteritems(): self.assertTrue(k in dst_headers) self.assertEqual(v, dst_headers[k]) self.assertFalse('new-owner' in dst_headers) def test_generate_request_headers_with_sysmeta(self): base = Controller(self.app) good_hdrs = {'x-base-sysmeta-foo': 'ok', 'X-Base-sysmeta-Bar': 'also ok'} bad_hdrs = {'x-base-sysmeta-': 'too short'} hdrs = dict(good_hdrs) hdrs.update(bad_hdrs) req = Request.blank('/v1/a/c/o', headers=hdrs) dst_headers = base.generate_request_headers(req, transfer=True) for k, v in good_hdrs.iteritems(): self.assertTrue(k.lower() in dst_headers) self.assertEqual(v, dst_headers[k.lower()]) for k, v in bad_hdrs.iteritems(): self.assertFalse(k.lower() in dst_headers) swift-1.13.1/test/unit/proxy/controllers/__init__.py0000664000175400017540000000000012323703611023651 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/proxy/test_mem_server.py0000664000175400017540000000251112323703611022760 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from test.unit.proxy import test_server from test.unit.proxy.test_server import teardown from swift.obj import mem_server def setup(): test_server.do_setup(mem_server) class TestController(test_server.TestController): pass class TestProxyServer(test_server.TestProxyServer): pass class TestObjectController(test_server.TestObjectController): pass class TestContainerController(test_server.TestContainerController): pass class TestAccountController(test_server.TestAccountController): pass class TestAccountControllerFakeGetResponse( test_server.TestAccountControllerFakeGetResponse): pass if __name__ == '__main__': setup() try: unittest.main() finally: teardown() swift-1.13.1/test/unit/proxy/__init__.py0000664000175400017540000000000012323703611021303 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/obj/0000775000175400017540000000000012323703665016606 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/obj/test_expirer.py0000664000175400017540000005760512323703611021701 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import urllib from time import time from unittest import main, TestCase from test.unit import FakeLogger from copy import deepcopy import mock from swift.common import internal_client from swift.obj import expirer def not_random(): return 0.5 last_not_sleep = 0 def not_sleep(seconds): global last_not_sleep last_not_sleep = seconds class TestObjectExpirer(TestCase): maxDiff = None def setUp(self): global not_sleep self.old_loadapp = internal_client.loadapp self.old_sleep = internal_client.sleep internal_client.loadapp = lambda *a, **kw: None internal_client.sleep = not_sleep def teardown(self): internal_client.sleep = self.old_sleep internal_client.loadapp = self.loadapp def test_get_process_values_from_kwargs(self): x = expirer.ObjectExpirer({}) vals = { 'processes': 5, 'process': 1, } self.assertEqual((5, 1), x.get_process_values(vals)) def test_get_process_values_from_config(self): vals = { 'processes': 5, 'process': 1, } x = expirer.ObjectExpirer(vals) self.assertEqual((5, 1), x.get_process_values({})) def test_get_process_values_negative_process(self): vals = { 'processes': 5, 'process': -1, } # from config x = expirer.ObjectExpirer(vals) self.assertRaises(ValueError, x.get_process_values, {}) # from kwargs x = expirer.ObjectExpirer({}) self.assertRaises(ValueError, x.get_process_values, vals) def test_get_process_values_negative_processes(self): vals = { 'processes': -5, 'process': 1, } # from config x = expirer.ObjectExpirer(vals) self.assertRaises(ValueError, x.get_process_values, {}) # from kwargs x = expirer.ObjectExpirer({}) self.assertRaises(ValueError, x.get_process_values, vals) def test_get_process_values_process_greater_than_processes(self): vals = { 'processes': 5, 'process': 7, } # from config x = expirer.ObjectExpirer(vals) self.assertRaises(ValueError, x.get_process_values, {}) # from kwargs x = expirer.ObjectExpirer({}) self.assertRaises(ValueError, x.get_process_values, vals) def test_init_concurrency_too_small(self): conf = { 'concurrency': 0, } self.assertRaises(ValueError, expirer.ObjectExpirer, conf) conf = { 'concurrency': -1, } self.assertRaises(ValueError, expirer.ObjectExpirer, conf) def test_process_based_concurrency(self): class ObjectExpirer(expirer.ObjectExpirer): def __init__(self, conf): super(ObjectExpirer, self).__init__(conf) self.processes = 3 self.deleted_objects = {} def delete_object(self, actual_obj, timestamp, container, obj): if container not in self.deleted_objects: self.deleted_objects[container] = set() self.deleted_objects[container].add(obj) class InternalClient(object): def __init__(self, containers): self.containers = containers def get_account_info(self, *a, **kw): return len(self.containers.keys()), \ sum([len(self.containers[x]) for x in self.containers]) def iter_containers(self, *a, **kw): return [{'name': x} for x in self.containers.keys()] def iter_objects(self, account, container): return [{'name': x} for x in self.containers[container]] def delete_container(*a, **kw): pass containers = { 0: set('1-one 2-two 3-three'.split()), 1: set('2-two 3-three 4-four'.split()), 2: set('5-five 6-six'.split()), 3: set('7-seven'.split()), } x = ObjectExpirer({}) x.swift = InternalClient(containers) deleted_objects = {} for i in xrange(3): x.process = i x.run_once() self.assertNotEqual(deleted_objects, x.deleted_objects) deleted_objects = deepcopy(x.deleted_objects) self.assertEqual(containers, deleted_objects) def test_delete_object(self): class InternalClient(object): def __init__(self, test, account, container, obj): self.test = test self.account = account self.container = container self.obj = obj self.delete_object_called = False def delete_object(self, account, container, obj): self.test.assertEqual(self.account, account) self.test.assertEqual(self.container, container) self.test.assertEqual(self.obj, obj) self.delete_object_called = True class DeleteActualObject(object): def __init__(self, test, actual_obj, timestamp): self.test = test self.actual_obj = actual_obj self.timestamp = timestamp self.called = False def __call__(self, actual_obj, timestamp): self.test.assertEqual(self.actual_obj, actual_obj) self.test.assertEqual(self.timestamp, timestamp) self.called = True container = 'container' obj = 'obj' actual_obj = 'actual_obj' timestamp = 'timestamp' x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = \ InternalClient(self, x.expiring_objects_account, container, obj) x.delete_actual_object = \ DeleteActualObject(self, actual_obj, timestamp) x.delete_object(actual_obj, timestamp, container, obj) self.assertTrue(x.swift.delete_object_called) self.assertTrue(x.delete_actual_object.called) def test_report(self): x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.report() self.assertEqual(x.logger.log_dict['info'], []) x.logger._clear() x.report(final=True) self.assertTrue('completed' in x.logger.log_dict['info'][-1][0][0], x.logger.log_dict['info']) self.assertTrue('so far' not in x.logger.log_dict['info'][-1][0][0], x.logger.log_dict['info']) x.logger._clear() x.report_last_time = time() - x.report_interval x.report() self.assertTrue('completed' not in x.logger.log_dict['info'][-1][0][0], x.logger.log_dict['info']) self.assertTrue('so far' in x.logger.log_dict['info'][-1][0][0], x.logger.log_dict['info']) def test_run_once_nothing_to_do(self): x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = 'throw error because a string does not have needed methods' x.run_once() self.assertEqual(x.logger.log_dict['exception'], [(("Unhandled exception",), {}, "'str' object has no attribute " "'get_account_info'")]) def test_run_once_calls_report(self): class InternalClient(object): def get_account_info(*a, **kw): return 1, 2 def iter_containers(*a, **kw): return [] x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = InternalClient() x.run_once() self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) def test_container_timestamp_break(self): class InternalClient(object): def __init__(self, containers): self.containers = containers def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def iter_objects(*a, **kw): raise Exception('This should not have been called') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = InternalClient([{'name': str(int(time() + 86400))}]) x.run_once() for exccall in x.logger.log_dict['exception']: self.assertTrue( 'This should not have been called' not in exccall[0][0]) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) # Reverse test to be sure it still would blow up the way expected. x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = InternalClient([{'name': str(int(time() - 86400))}]) x.run_once() self.assertEqual( x.logger.log_dict['exception'], [(('Unhandled exception',), {}, str(Exception('This should not have been called')))]) def test_object_timestamp_break(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects def should_not_be_called(*a, **kw): raise Exception('This should not have been called') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % int(time() + 86400)}]) x.run_once() for exccall in x.logger.log_dict['exception']: self.assertTrue( 'This should not have been called' not in exccall[0][0]) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) # Reverse test to be sure it still would blow up the way expected. x = expirer.ObjectExpirer({}) x.logger = FakeLogger() ts = int(time() - 86400) x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) x.delete_actual_object = should_not_be_called x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEqual( excswhiledeleting, ['Exception while deleting object %d %d-actual-obj ' 'This should not have been called' % (ts, ts)]) def test_failed_delete_keeps_entry(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def delete_object(*a, **kw): raise Exception('This should not have been called') def iter_objects(self, *a, **kw): return self.objects def deliberately_blow_up(actual_obj, timestamp): raise Exception('failed to delete actual object') def should_not_get_called(container, obj): raise Exception('This should not have been called') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.iter_containers = lambda: [str(int(time() - 86400))] ts = int(time() - 86400) x.delete_actual_object = deliberately_blow_up x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEqual( excswhiledeleting, ['Exception while deleting object %d %d-actual-obj ' 'failed to delete actual object' % (ts, ts)]) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) # Reverse test to be sure it still would blow up the way expected. x = expirer.ObjectExpirer({}) x.logger = FakeLogger() ts = int(time() - 86400) x.delete_actual_object = lambda o, t: None x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % ts}]) x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEqual( excswhiledeleting, ['Exception while deleting object %d %d-actual-obj This should ' 'not have been called' % (ts, ts)]) def test_success_gets_counted(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.delete_actual_object = lambda o, t: None self.assertEqual(x.report_objects, 0) x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': '%d-actual-obj' % int(time() - 86400)}]) x.run_once() self.assertEqual(x.report_objects, 1) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 1 objects expired',), {})]) def test_delete_actual_object_does_not_get_unicode(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): pass def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects got_unicode = [False] def delete_actual_object_test_for_unicode(actual_obj, timestamp): if isinstance(actual_obj, unicode): got_unicode[0] = True x = expirer.ObjectExpirer({}) x.logger = FakeLogger() x.delete_actual_object = delete_actual_object_test_for_unicode self.assertEqual(x.report_objects, 0) x.swift = InternalClient( [{'name': str(int(time() - 86400))}], [{'name': u'%d-actual-obj' % int(time() - 86400)}]) x.run_once() self.assertEqual(x.report_objects, 1) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 1 objects expired',), {})]) self.assertFalse(got_unicode[0]) def test_failed_delete_continues_on(self): class InternalClient(object): def __init__(self, containers, objects): self.containers = containers self.objects = objects def get_account_info(*a, **kw): return 1, 2 def iter_containers(self, *a, **kw): return self.containers def delete_container(*a, **kw): raise Exception('failed to delete container') def delete_object(*a, **kw): pass def iter_objects(self, *a, **kw): return self.objects def fail_delete_actual_object(actual_obj, timestamp): raise Exception('failed to delete actual object') x = expirer.ObjectExpirer({}) x.logger = FakeLogger() cts = int(time() - 86400) ots = int(time() - 86400) containers = [ {'name': str(cts)}, {'name': str(cts + 1)}, ] objects = [ {'name': '%d-actual-obj' % ots}, {'name': '%d-next-obj' % ots} ] x.swift = InternalClient(containers, objects) x.delete_actual_object = fail_delete_actual_object x.run_once() excswhiledeleting = [] for exccall in x.logger.log_dict['exception']: if exccall[0][0].startswith('Exception while deleting '): excswhiledeleting.append(exccall[0][0]) self.assertEqual(sorted(excswhiledeleting), sorted([ 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object' % (cts, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object' % (cts, ots), 'Exception while deleting object %d %d-actual-obj failed to ' 'delete actual object' % (cts + 1, ots), 'Exception while deleting object %d %d-next-obj failed to ' 'delete actual object' % (cts + 1, ots), 'Exception while deleting container %d failed to delete ' 'container' % (cts,), 'Exception while deleting container %d failed to delete ' 'container' % (cts + 1,)])) self.assertEqual( x.logger.log_dict['info'], [(('Pass beginning; 1 possible containers; ' '2 possible objects',), {}), (('Pass completed in 0s; 0 objects expired',), {})]) def test_run_forever_initial_sleep_random(self): global last_not_sleep def raise_system_exit(): raise SystemExit('test_run_forever') interval = 1234 x = expirer.ObjectExpirer({'__file__': 'unit_test', 'interval': interval}) orig_random = expirer.random orig_sleep = expirer.sleep try: expirer.random = not_random expirer.sleep = not_sleep x.run_once = raise_system_exit x.run_forever() except SystemExit as err: pass finally: expirer.random = orig_random expirer.sleep = orig_sleep self.assertEqual(str(err), 'test_run_forever') self.assertEqual(last_not_sleep, 0.5 * interval) def test_run_forever_catches_usual_exceptions(self): raises = [0] def raise_exceptions(): raises[0] += 1 if raises[0] < 2: raise Exception('exception %d' % raises[0]) raise SystemExit('exiting exception %d' % raises[0]) x = expirer.ObjectExpirer({}) x.logger = FakeLogger() orig_sleep = expirer.sleep try: expirer.sleep = not_sleep x.run_once = raise_exceptions x.run_forever() except SystemExit as err: pass finally: expirer.sleep = orig_sleep self.assertEqual(str(err), 'exiting exception 2') self.assertEqual(x.logger.log_dict['exception'], [(('Unhandled exception',), {}, 'exception 1')]) def test_delete_actual_object(self): got_env = [None] def fake_app(env, start_response): got_env[0] = env start_response('204 No Content', [('Content-Length', '0')]) return [] internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' x.delete_actual_object('/path/to/object', ts) self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts) def test_delete_actual_object_nourlquoting(self): # delete_actual_object should not do its own url quoting because # internal client's make_request handles that. got_env = [None] def fake_app(env, start_response): got_env[0] = env start_response('204 No Content', [('Content-Length', '0')]) return [] internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) ts = '1234' x.delete_actual_object('/path/to/object name', ts) self.assertEqual(got_env[0]['HTTP_X_IF_DELETE_AT'], ts) self.assertEqual(got_env[0]['PATH_INFO'], '/v1/path/to/object name') def test_delete_actual_object_handles_404(self): def fake_app(env, start_response): start_response('404 Not Found', [('Content-Length', '0')]) return [] internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') def test_delete_actual_object_handles_412(self): def fake_app(env, start_response): start_response('412 Precondition Failed', [('Content-Length', '0')]) return [] internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) x.delete_actual_object('/path/to/object', '1234') def test_delete_actual_object_does_not_handle_odd_stuff(self): def fake_app(env, start_response): start_response( '503 Internal Server Error', [('Content-Length', '0')]) return [] internal_client.loadapp = lambda *a, **kw: fake_app x = expirer.ObjectExpirer({}) exc = None try: x.delete_actual_object('/path/to/object', '1234') except Exception as err: exc = err finally: pass self.assertEqual(503, exc.resp.status_int) def test_delete_actual_object_quotes(self): name = 'this name should get quoted' timestamp = '1366063156.863045' x = expirer.ObjectExpirer({}) x.swift.make_request = mock.MagicMock() x.delete_actual_object(name, timestamp) x.swift.make_request.assert_called_once() self.assertEqual(x.swift.make_request.call_args[0][1], '/v1/' + urllib.quote(name)) if __name__ == '__main__': main() swift-1.13.1/test/unit/obj/test_server.py0000775000175400017540000045653412323703614021543 0ustar jenkinsjenkins00000000000000#-*- coding:utf-8 -*- # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.obj.server""" import cPickle as pickle import datetime import operator import os import mock import unittest import math from shutil import rmtree from StringIO import StringIO from time import gmtime, strftime, time, struct_time from tempfile import mkdtemp from hashlib import md5 from eventlet import sleep, spawn, wsgi, listen, Timeout, tpool from nose import SkipTest from test.unit import FakeLogger, debug_logger from test.unit import connect_tcp, readuntil2crlfs from swift.obj import server as object_server from swift.obj import diskfile from swift.common import utils from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ NullLogger, storage_directory, public, replication from swift.common import constraints from swift.common.swob import Request, HeaderKeyDict from swift.common.exceptions import DiskFileDeviceUnavailable def mock_time(*args, **kwargs): return 5000.0 class TestObjectController(unittest.TestCase): """Test swift.obj.server.ObjectController""" def setUp(self): """Set up for testing swift.object.server.ObjectController""" utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' self.testdir = \ os.path.join(mkdtemp(), 'tmp_test_object_server_ObjectController') mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) conf = {'devices': self.testdir, 'mount_check': 'false'} self.object_controller = object_server.ObjectController( conf, logger=debug_logger()) self.object_controller.bytes_per_sync = 1 self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) self.df_mgr = diskfile.DiskFileManager(conf, self.object_controller.logger) def tearDown(self): """Tear down for testing swift.object.server.ObjectController""" rmtree(os.path.dirname(self.testdir)) tpool.execute = self._orig_tpool_exc def check_all_api_methods(self, obj_name='o', alt_res=None): path = '/sda1/p/a/c/%s' % obj_name body = 'SPECIAL_STRING' op_table = { "PUT": (body, alt_res or 201, ''), # create one "GET": ('', alt_res or 200, body), # check it "POST": ('', alt_res or 202, ''), # update it "HEAD": ('', alt_res or 200, ''), # head it "DELETE": ('', alt_res or 204, '') # delete it } for method in ["PUT", "GET", "POST", "HEAD", "DELETE"]: in_body, res, out_body = op_table[method] timestamp = normalize_timestamp(time()) req = Request.blank( path, environ={'REQUEST_METHOD': method}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = in_body resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, res) if out_body and (200 <= res < 300): self.assertEqual(resp.body, out_body) def test_REQUEST_SPECIAL_CHARS(self): obj = 'special昆%20/%' self.check_all_api_methods(obj) def test_device_unavailable(self): def raise_disk_unavail(*args, **kwargs): raise DiskFileDeviceUnavailable() self.object_controller.get_diskfile = raise_disk_unavail self.check_all_api_methods(alt_res=507) def test_allowed_headers(self): dah = ['content-disposition', 'content-encoding', 'x-delete-at', 'x-object-manifest', 'x-static-large-object'] conf = {'devices': self.testdir, 'mount_check': 'false', 'allowed_headers': ','.join(['content-type'] + dah)} self.object_controller = object_server.ObjectController( conf, logger=debug_logger()) self.assertEqual(self.object_controller.allowed_headers, set(dah)) def test_POST_update_meta(self): # Test swift.obj.server.ObjectController.POST original_headers = self.object_controller.allowed_headers test_headers = 'content-encoding foo bar'.split() self.object_controller.allowed_headers = set(test_headers) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'Foo': 'fooheader', 'Baz': 'bazheader', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', 'Content-Encoding': 'gzip', 'Foo': 'fooheader', 'Bar': 'barheader', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assert_("X-Object-Meta-1" not in resp.headers and "X-Object-Meta-Two" not in resp.headers and "X-Object-Meta-3" in resp.headers and "X-Object-Meta-4" in resp.headers and "Foo" in resp.headers and "Bar" in resp.headers and "Baz" not in resp.headers and "Content-Encoding" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assert_("X-Object-Meta-1" not in resp.headers and "X-Object-Meta-Two" not in resp.headers and "X-Object-Meta-3" in resp.headers and "X-Object-Meta-4" in resp.headers and "Foo" in resp.headers and "Bar" in resp.headers and "Baz" not in resp.headers and "Content-Encoding" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assert_("X-Object-Meta-3" not in resp.headers and "X-Object-Meta-4" not in resp.headers and "Foo" not in resp.headers and "Bar" not in resp.headers and "Content-Encoding" not in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') # test defaults self.object_controller.allowed_headers = original_headers timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'Foo': 'fooheader', 'X-Object-Meta-1': 'One', 'X-Object-Manifest': 'c/bar', 'Content-Encoding': 'gzip', 'Content-Disposition': 'bar', }) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assert_("X-Object-Meta-1" in resp.headers and "Foo" not in resp.headers and "Content-Encoding" in resp.headers and "X-Object-Manifest" in resp.headers and "Content-Disposition" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-3': 'Three', 'Foo': 'fooheader', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assert_("X-Object-Meta-1" not in resp.headers and "Foo" not in resp.headers and "Content-Encoding" not in resp.headers and "X-Object-Manifest" not in resp.headers and "Content-Disposition" not in resp.headers and "X-Object-Meta-3" in resp.headers) self.assertEquals(resp.headers['Content-Type'], 'application/x-test') # Test for empty metadata timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'X-Object-Meta-3': ''}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 202) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assertEquals(resp.headers["x-object-meta-3"], '') def test_POST_old_timestamp(self): ts = time() timestamp = normalize_timestamp(ts) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) # Same timestamp should result in 409 req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-3': 'Three', 'X-Object-Meta-4': 'Four', 'Content-Encoding': 'gzip', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) # Earlier timestamp should result in 409 timestamp = normalize_timestamp(ts - 1) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-5': 'Five', 'X-Object-Meta-6': 'Six', 'Content-Encoding': 'gzip', 'Content-Type': 'application/x-test'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) def test_POST_not_exist(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-1': 'One', 'X-Object-Meta-2': 'Two', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) def test_POST_invalid_path(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, 'X-Object-Meta-1': 'One', 'X-Object-Meta-2': 'Two', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_POST_no_timestamp(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Object-Meta-1': 'One', 'X-Object-Meta-2': 'Two', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 400) def test_POST_bad_timestamp(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': 'bad', 'X-Object-Meta-1': 'One', 'X-Object-Meta-2': 'Two', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 400) def test_POST_container_connection(self): def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): self.status = status self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' self.with_exc = with_exc def getresponse(self): if self.with_exc: raise Exception('test') return self def read(self, amt=None): return '' return lambda *args, **kwargs: FakeConn(response, with_exc) old_http_connect = object_server.http_connect try: ts = time() timestamp = normalize_timestamp(ts) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Length': '0'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(ts + 1), 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new1'}) object_server.http_connect = mock_http_connect(202) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(ts + 2), 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new1'}) object_server.http_connect = mock_http_connect(202, with_exc=True) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(ts + 3), 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new2'}) object_server.http_connect = mock_http_connect(500) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) finally: object_server.http_connect = old_http_connect def test_POST_quarantine_zbyte(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') objfile.open() file_name = os.path.basename(objfile._data_file) with open(objfile._data_file) as fp: metadata = diskfile.read_metadata(fp) os.unlink(objfile._data_file) with open(objfile._data_file, 'w') as fp: diskfile.write_metadata(fp, metadata) self.assertEquals(os.listdir(objfile._datadir)[0], file_name) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(time())}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(objfile._data_file))) self.assertEquals(os.listdir(quar_dir)[0], file_name) def test_PUT_invalid_path(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_PUT_no_timestamp(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'CONTENT_LENGTH': '0'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_PUT_no_content_type(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '6'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_PUT_invalid_content_type(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '6', 'Content-Type': '\xff\xff'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) self.assert_('Content-Type' in resp.body) def test_PUT_no_content_length(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' del req.headers['Content-Length'] resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 411) def test_PUT_zero_content_length(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream'}) req.body = '' self.assertEquals(req.headers['Content-Length'], '0') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) def test_PUT_bad_transfer_encoding(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' req.headers['Transfer-Encoding'] = 'bad' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 400) def test_PUT_if_none_match_star(self): # First PUT should succeed timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'If-None-Match': '*'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) # File should already exist so it should fail timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'If-None-Match': '*'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) def test_PUT_if_none_match(self): # PUT with if-none-match set and nothing there should succede timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'If-None-Match': 'notthere'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) # PUT with if-none-match of the object etag should fail timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'If-None-Match': '0b4c12d7e0a73840c1c4f148fda3b037'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) def test_PUT_common(self): timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY') self.assertEquals(diskfile.read_metadata(objfile), {'X-Timestamp': timestamp, 'Content-Length': '6', 'ETag': '0b4c12d7e0a73840c1c4f148fda3b037', 'Content-Type': 'application/octet-stream', 'name': '/a/c/o'}) def test_PUT_overwrite(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY TWO') self.assertEquals(diskfile.read_metadata(objfile), {'X-Timestamp': timestamp, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', 'name': '/a/c/o', 'Content-Encoding': 'gzip'}) def test_PUT_overwrite_w_delete_at(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'X-Delete-At': 9999999999, 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 201) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assertTrue(os.path.isfile(objfile)) self.assertEqual(open(objfile).read(), 'VERIFY TWO') self.assertEqual(diskfile.read_metadata(objfile), {'X-Timestamp': timestamp, 'Content-Length': '10', 'ETag': 'b381a4c5dab1eaa1eb9711fa647cd039', 'Content-Type': 'text/plain', 'name': '/a/c/o', 'Content-Encoding': 'gzip'}) def test_PUT_old_timestamp(self): ts = time() req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(ts), 'Content-Length': '6', 'Content-Type': 'application/octet-stream'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(ts), 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY TWO' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(ts - 1), 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}) req.body = 'VERIFY THREE' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) def test_PUT_no_etag(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'text/plain'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) def test_PUT_invalid_etag(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'text/plain', 'ETag': 'invalid'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 422) def test_PUT_user_metadata(self): timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY THREE' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assert_(os.path.isfile(objfile)) self.assertEquals(open(objfile).read(), 'VERIFY THREE') self.assertEquals(diskfile.read_metadata(objfile), {'X-Timestamp': timestamp, 'Content-Length': '12', 'ETag': 'b114ab7b90d9ccac4bd5d99cc7ebb568', 'Content-Type': 'text/plain', 'name': '/a/c/o', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) def test_PUT_client_timeout(self): class FakeTimeout(BaseException): def __enter__(self): raise self def __exit__(self, typ, value, tb): pass # This is just so the test fails when run on older object server code # instead of exploding. if not hasattr(object_server, 'ChunkReadTimeout'): object_server.ChunkReadTimeout = None with mock.patch.object(object_server, 'ChunkReadTimeout', FakeTimeout): timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'text/plain', 'Content-Length': '6'}) req.environ['wsgi.input'] = StringIO('VERIFY') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 408) def test_PUT_container_connection(self): def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): self.status = status self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' self.with_exc = with_exc def getresponse(self): if self.with_exc: raise Exception('test') return self def read(self, amt=None): return '' return lambda *args, **kwargs: FakeConn(response, with_exc) old_http_connect = object_server.http_connect try: timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new1', 'Content-Length': '0'}) object_server.http_connect = mock_http_connect(201) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new1', 'Content-Length': '0'}) object_server.http_connect = mock_http_connect(500) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'X-Container-Host': '1.2.3.4:0', 'X-Container-Partition': '3', 'X-Container-Device': 'sda1', 'X-Container-Timestamp': '1', 'Content-Type': 'application/new1', 'Content-Length': '0'}) object_server.http_connect = mock_http_connect(500, with_exc=True) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) finally: object_server.http_connect = old_http_connect def test_HEAD(self): # Test swift.obj.server.ObjectController.HEAD req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) timestamp = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.content_length, 6) self.assertEquals(resp.content_type, 'application/x-test') self.assertEquals(resp.headers['content-type'], 'application/x-test') self.assertEquals( resp.headers['last-modified'], strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(math.ceil(float(timestamp))))) self.assertEquals(resp.headers['etag'], '"0b4c12d7e0a73840c1c4f148fda3b037"') self.assertEquals(resp.headers['x-object-meta-1'], 'One') self.assertEquals(resp.headers['x-object-meta-two'], 'Two') objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-length': '6'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) def test_HEAD_quarantine_zbyte(self): # Test swift.obj.server.ObjectController.GET timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') disk_file.open() file_name = os.path.basename(disk_file._data_file) with open(disk_file._data_file) as fp: metadata = diskfile.read_metadata(fp) os.unlink(disk_file._data_file) with open(disk_file._data_file, 'w') as fp: diskfile.write_metadata(fp, metadata) file_name = os.path.basename(disk_file._data_file) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) self.assertEquals(os.listdir(quar_dir)[0], file_name) def test_GET(self): # Test swift.obj.server.ObjectController.GET req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test', 'X-Object-Meta-1': 'One', 'X-Object-Meta-Two': 'Two'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body, 'VERIFY') self.assertEquals(resp.content_length, 6) self.assertEquals(resp.content_type, 'application/x-test') self.assertEquals(resp.headers['content-length'], '6') self.assertEquals(resp.headers['content-type'], 'application/x-test') self.assertEquals( resp.headers['last-modified'], strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(math.ceil(float(timestamp))))) self.assertEquals(resp.headers['etag'], '"0b4c12d7e0a73840c1c4f148fda3b037"') self.assertEquals(resp.headers['x-object-meta-1'], 'One') self.assertEquals(resp.headers['x-object-meta-two'], 'Two') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req.range = 'bytes=1-3' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 206) self.assertEquals(resp.body, 'ERI') self.assertEquals(resp.headers['content-length'], '3') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req.range = 'bytes=1-' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 206) self.assertEquals(resp.body, 'ERIFY') self.assertEquals(resp.headers['content-length'], '5') req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req.range = 'bytes=-2' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 206) self.assertEquals(resp.body, 'FY') self.assertEquals(resp.headers['content-length'], '2') objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') os.unlink(objfile) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application:octet-stream', 'Content-Length': '6'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) def test_GET_if_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) etag = resp.etag req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Match': '"11111111111111111111111111111111"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={ 'If-Match': '"11111111111111111111111111111111", "%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={ 'If-Match': '"11111111111111111111111111111111", ' '"22222222222222222222222222222222"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) def test_HEAD_if_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) etag = resp.etag req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-Match': '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-Match': '"11111111111111111111111111111111"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'If-Match': '"11111111111111111111111111111111", "%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={ 'If-Match': '"11111111111111111111111111111111", ' '"22222222222222222222222222222222"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) def test_GET_if_none_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) etag = resp.etag req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': '"11111111111111111111111111111111"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-None-Match': '"11111111111111111111111111111111", ' '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) def test_HEAD_if_none_match(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) etag = resp.etag req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': '*'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': '"11111111111111111111111111111111"'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.etag, etag) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'If-None-Match': '"11111111111111111111111111111111", ' '"%s"' % etag}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) self.assertEquals(resp.etag, etag) def test_GET_if_modified_since(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) since = \ strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 1)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) since = \ strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) since = resp.headers['Last-Modified'] self.assertEquals(since, strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(math.ceil(float(timestamp))))) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': since}) resp = self.object_controller.GET(req) self.assertEquals(resp.status_int, 304) timestamp = normalize_timestamp(int(time())) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp))) req = Request.blank('/sda1/p/a/c/o2', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 304) def test_GET_if_unmodified_since(self): timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4'}) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) since = strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 1)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) since = \ strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) - 9)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) since = \ strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(float(timestamp) + 9)) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': since}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) since = resp.headers['Last-Modified'] self.assertEquals(since, strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(math.ceil(float(timestamp))))) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': since}) resp = self.object_controller.GET(req) self.assertEquals(resp.status_int, 200) def test_GET_quarantine(self): # Test swift.obj.server.ObjectController.GET timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') disk_file.open() file_name = os.path.basename(disk_file._data_file) etag = md5() etag.update('VERIF') etag = etag.hexdigest() metadata = {'X-Timestamp': timestamp, 'name': '/a/c/o', 'Content-Length': 6, 'ETag': etag} diskfile.write_metadata(disk_file._fp, metadata) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) body = resp.body # actually does quarantining self.assertEquals(body, 'VERIFY') self.assertEquals(os.listdir(quar_dir)[0], file_name) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) def test_GET_quarantine_zbyte(self): # Test swift.obj.server.ObjectController.GET timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') disk_file.open() file_name = os.path.basename(disk_file._data_file) with open(disk_file._data_file) as fp: metadata = diskfile.read_metadata(fp) os.unlink(disk_file._data_file) with open(disk_file._data_file, 'w') as fp: diskfile.write_metadata(fp, metadata) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) self.assertEquals(os.listdir(quar_dir)[0], file_name) def test_GET_quarantine_range(self): # Test swift.obj.server.ObjectController.GET timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Type': 'application/x-test'}) req.body = 'VERIFY' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) disk_file = self.df_mgr.get_diskfile('sda1', 'p', 'a', 'c', 'o') disk_file.open() file_name = os.path.basename(disk_file._data_file) etag = md5() etag.update('VERIF') etag = etag.hexdigest() metadata = {'X-Timestamp': timestamp, 'name': '/a/c/o', 'Content-Length': 6, 'ETag': etag} diskfile.write_metadata(disk_file._fp, metadata) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) req = Request.blank('/sda1/p/a/c/o') req.range = 'bytes=0-4' # partial resp = req.get_response(self.object_controller) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) resp.body self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) self.assertFalse(os.path.isdir(quar_dir)) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) req = Request.blank('/sda1/p/a/c/o') req.range = 'bytes=1-6' # partial resp = req.get_response(self.object_controller) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) resp.body self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) self.assertFalse(os.path.isdir(quar_dir)) req = Request.blank('/sda1/p/a/c/o') req.range = 'bytes=0-14' # full resp = req.get_response(self.object_controller) quar_dir = os.path.join( self.testdir, 'sda1', 'quarantined', 'objects', os.path.basename(os.path.dirname(disk_file._data_file))) self.assertEquals(os.listdir(disk_file._datadir)[0], file_name) resp.body self.assertTrue(os.path.isdir(quar_dir)) req = Request.blank('/sda1/p/a/c/o') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) @mock.patch("time.time", mock_time) def test_DELETE(self): # Test swift.obj.server.ObjectController.DELETE req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) # self.assertRaises(KeyError, self.object_controller.DELETE, req) # The following should have created a tombstone file timestamp = normalize_timestamp(1000) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) ts_1000_file = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertTrue(os.path.isfile(ts_1000_file)) # There should now be a 1000 ts file. self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) # The following should *not* have created a tombstone file. timestamp = normalize_timestamp(999) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) ts_999_file = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertFalse(os.path.isfile(ts_999_file)) self.assertTrue(os.path.isfile(ts_1000_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1000_file))), 1) timestamp = normalize_timestamp(1002) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', }) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) # There should now be 1000 ts and a 1001 data file. data_1002_file = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.data') self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(data_1002_file))), 1) # The following should *not* have created a tombstone file. timestamp = normalize_timestamp(1001) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) ts_1001_file = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertFalse(os.path.isfile(ts_1001_file)) self.assertTrue(os.path.isfile(data_1002_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1001_file))), 1) timestamp = normalize_timestamp(1003) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) ts_1003_file = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertTrue(os.path.isfile(ts_1003_file)) self.assertEquals(len(os.listdir(os.path.dirname(ts_1003_file))), 1) def test_DELETE_container_updates(self): # Test swift.obj.server.ObjectController.DELETE and container # updates, making sure container update is called in the correct # state. timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': timestamp, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', }) req.body = 'test' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) calls_made = [0] def our_container_update(*args, **kwargs): calls_made[0] += 1 orig_cu = self.object_controller.container_update self.object_controller.container_update = our_container_update try: # The following request should return 409 (HTTP Conflict). A # tombstone file should not have been created with this timestamp. timestamp = normalize_timestamp(float(timestamp) - 1) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 409) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) self.assertEquals(0, calls_made[0]) # The following request should return 204, and the object should # be truly deleted (container update is performed) because this # timestamp is newer. A tombstone file should have been created # with this timestamp. sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(1, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) # The following request should return a 404, as the object should # already have been deleted, but it should have also performed a # container update because the timestamp is newer, and a tombstone # file should also exist with this timestamp. sleep(.00001) timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assert_(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) # The following request should return a 404, as the object should # already have been deleted, and it should not have performed a # container update because the timestamp is older, or created a # tombstone file with this timestamp. timestamp = normalize_timestamp(float(timestamp) - 1) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), timestamp + '.ts') self.assertFalse(os.path.isfile(objfile)) self.assertEquals(2, calls_made[0]) self.assertEquals(len(os.listdir(os.path.dirname(objfile))), 1) finally: self.object_controller.container_update = orig_cu def test_call_bad_request(self): # Test swift.obj.server.ObjectController.__call__ inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) self.object_controller.__call__({'REQUEST_METHOD': 'PUT', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '400 ') def test_call_not_found(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) self.object_controller.__call__({'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '404 ') def test_call_bad_method(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) self.object_controller.__call__({'REQUEST_METHOD': 'INVALID', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_call_name_collision(self): def my_check(*args): return False def my_hash_path(*args): return md5('collide').hexdigest() with mock.patch("swift.obj.diskfile.hash_path", my_hash_path): with mock.patch("swift.obj.server.check_object_creation", my_check): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) self.object_controller.__call__({ 'REQUEST_METHOD': 'PUT', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'CONTENT_TYPE': 'text/html', 'HTTP_X_TIMESTAMP': normalize_timestamp(1.2), 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '201 ') inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() def start_response(*args): """Sends args to outbuf""" outbuf.writelines(args) self.object_controller.__call__({ 'REQUEST_METHOD': 'PUT', 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/b/d/x', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'CONTENT_TYPE': 'text/html', 'HTTP_X_TIMESTAMP': normalize_timestamp(1.3), 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '403 ') def test_invalid_method_doesnt_exist(self): errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.object_controller.__call__({ 'REQUEST_METHOD': 'method_doesnt_exist', 'PATH_INFO': '/sda1/p/a/c/o'}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_invalid_method_is_not_public(self): errbuf = StringIO() outbuf = StringIO() def start_response(*args): outbuf.writelines(args) self.object_controller.__call__({'REQUEST_METHOD': '__init__', 'PATH_INFO': '/sda1/p/a/c/o'}, start_response) self.assertEquals(errbuf.getvalue(), '') self.assertEquals(outbuf.getvalue()[:4], '405 ') def test_chunked_put(self): listener = listen(('localhost', 0)) port = listener.getsockname()[1] killer = spawn(wsgi.server, listener, self.object_controller, NullLogger()) sock = connect_tcp(('localhost', port)) fd = sock.makefile() fd.write('PUT /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' 'Content-Type: text/plain\r\n' 'Connection: close\r\nX-Timestamp: %s\r\n' 'Transfer-Encoding: chunked\r\n\r\n' '2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n' % normalize_timestamp( 1.0)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', port)) fd = sock.makefile() fd.write('GET /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) response = fd.read() self.assertEquals(response, 'oh hai') killer.kill() def test_chunked_content_length_mismatch_zero(self): listener = listen(('localhost', 0)) port = listener.getsockname()[1] killer = spawn(wsgi.server, listener, self.object_controller, NullLogger()) sock = connect_tcp(('localhost', port)) fd = sock.makefile() fd.write('PUT /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' 'Content-Type: text/plain\r\n' 'Connection: close\r\nX-Timestamp: %s\r\n' 'Content-Length: 0\r\n' 'Transfer-Encoding: chunked\r\n\r\n' '2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n' % normalize_timestamp( 1.0)) fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' self.assertEquals(headers[:len(exp)], exp) sock = connect_tcp(('localhost', port)) fd = sock.makefile() fd.write('GET /sda1/p/a/c/o HTTP/1.1\r\nHost: localhost\r\n' 'Connection: close\r\n\r\n') fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 200' self.assertEquals(headers[:len(exp)], exp) response = fd.read() self.assertEquals(response, 'oh hai') killer.kill() def test_max_object_name_length(self): timestamp = normalize_timestamp(time()) max_name_len = constraints.MAX_OBJECT_NAME_LENGTH req = Request.blank( '/sda1/p/a/c/' + ('1' * max_name_len), environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'DATA' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/' + ('2' * (max_name_len + 1)), environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'DATA' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_max_upload_time(self): class SlowBody(object): def __init__(self): self.sent = 0 def read(self, size=-1): if self.sent < 4: sleep(0.1) self.sent += 1 return ' ' return '' req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) self.object_controller.max_upload_time = 0.1 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 408) def test_short_body(self): class ShortBody(object): def __init__(self): self.sent = False def read(self, size=-1): if not self.sent: self.sent = True return ' ' return '' req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': ShortBody()}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 499) def test_bad_sinces(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': 'Not a valid date'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Modified-Since': 'Not a valid date'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) too_big_date_list = list(datetime.datetime.max.timetuple()) too_big_date_list[0] += 1 # bump up the year too_big_date = strftime( "%a, %d %b %Y %H:%M:%S UTC", struct_time(too_big_date_list)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'If-Unmodified-Since': too_big_date}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) def test_content_encoding(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain', 'Content-Encoding': 'gzip'}, body=' ') resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-encoding'], 'gzip') req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['content-encoding'], 'gzip') def test_async_update_http_connect(self): given_args = [] def fake_http_connect(*args): given_args.extend(args) raise Exception('test') orig_http_connect = object_server.http_connect try: object_server.http_connect = fake_http_connect self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': 'set'}, 'sda1') finally: object_server.http_connect = orig_http_connect self.assertEquals( given_args, ['127.0.0.1', '1234', 'sdc1', 1, 'PUT', '/a/c/o', { 'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'obj-server %s' % os.getpid()}]) def test_updating_multiple_delete_at_container_servers(self): self.object_controller.expiring_objects_account = 'exp' self.object_controller.expiring_objects_container_divisor = 60 http_connect_args = [] def fake_http_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): class SuccessfulFakeConn(object): @property def status(self): return 200 def getresponse(self): return self def read(self): return '' captured_args = {'ipaddr': ipaddr, 'port': port, 'device': device, 'partition': partition, 'method': method, 'path': path, 'ssl': ssl, 'headers': headers, 'query_string': query_string} http_connect_args.append( dict((k, v) for k, v in captured_args.iteritems() if v is not None)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '12345', 'Content-Type': 'application/burrito', 'Content-Length': '0', 'X-Container-Partition': '20', 'X-Container-Host': '1.2.3.4:5', 'X-Container-Device': 'sdb1', 'X-Delete-At': 9999999999, 'X-Delete-At-Container': '9999999960', 'X-Delete-At-Host': "10.1.1.1:6001,10.2.2.2:6002", 'X-Delete-At-Partition': '6237', 'X-Delete-At-Device': 'sdp,sdq'}) orig_http_connect = object_server.http_connect try: object_server.http_connect = fake_http_connect resp = req.get_response(self.object_controller) finally: object_server.http_connect = orig_http_connect self.assertEqual(resp.status_int, 201) http_connect_args.sort(key=operator.itemgetter('ipaddr')) self.assertEquals(len(http_connect_args), 3) self.assertEquals( http_connect_args[0], {'ipaddr': '1.2.3.4', 'port': '5', 'path': '/a/c/o', 'device': 'sdb1', 'partition': '20', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '10.1.1.1', 'port': '6001', 'path': '/exp/9999999960/9999999999-a/c/o', 'device': 'sdp', 'partition': '6237', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[2], {'ipaddr': '10.2.2.2', 'port': '6002', 'path': '/exp/9999999960/9999999999-a/c/o', 'device': 'sdq', 'partition': '6237', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), 'x-trans-id': '-'})}) def test_updating_multiple_container_servers(self): http_connect_args = [] def fake_http_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): class SuccessfulFakeConn(object): @property def status(self): return 200 def getresponse(self): return self def read(self): return '' captured_args = {'ipaddr': ipaddr, 'port': port, 'device': device, 'partition': partition, 'method': method, 'path': path, 'ssl': ssl, 'headers': headers, 'query_string': query_string} http_connect_args.append( dict((k, v) for k, v in captured_args.iteritems() if v is not None)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': '12345', 'Content-Type': 'application/burrito', 'Content-Length': '0', 'X-Container-Partition': '20', 'X-Container-Host': '1.2.3.4:5, 6.7.8.9:10', 'X-Container-Device': 'sdb1, sdf1'}) orig_http_connect = object_server.http_connect try: object_server.http_connect = fake_http_connect self.object_controller.PUT(req) finally: object_server.http_connect = orig_http_connect http_connect_args.sort(key=operator.itemgetter('ipaddr')) self.assertEquals(len(http_connect_args), 2) self.assertEquals( http_connect_args[0], {'ipaddr': '1.2.3.4', 'port': '5', 'path': '/a/c/o', 'device': 'sdb1', 'partition': '20', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '6.7.8.9', 'port': '10', 'path': '/a/c/o', 'device': 'sdf1', 'partition': '20', 'method': 'PUT', 'ssl': False, 'headers': HeaderKeyDict({ 'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', 'referer': 'PUT http://localhost/sda1/p/a/c/o', 'user-agent': 'obj-server %d' % os.getpid(), 'x-trans-id': '-'})}) def test_async_update_saves_on_exception(self): _prefix = utils.HASH_PATH_PREFIX utils.HASH_PATH_PREFIX = '' def fake_http_connect(*args): raise Exception('test') orig_http_connect = object_server.http_connect try: object_server.http_connect = fake_http_connect self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': 'set'}, 'sda1') finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), {'headers': {'x-timestamp': '1', 'x-out': 'set', 'user-agent': 'obj-server %s' % os.getpid()}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): _prefix = utils.HASH_PATH_PREFIX utils.HASH_PATH_PREFIX = '' def fake_http_connect(status): class FakeConn(object): def __init__(self, status): self.status = status def getresponse(self): return self def read(self): return '' return lambda *args: FakeConn(status) orig_http_connect = object_server.http_connect try: for status in (199, 300, 503): object_server.http_connect = fake_http_connect(status) self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': str(status)}, 'sda1') self.assertEquals( pickle.load(open(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), {'headers': {'x-timestamp': '1', 'x-out': str(status), 'user-agent': 'obj-server %s' % os.getpid()}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix def test_async_update_does_not_save_on_2xx(self): _prefix = utils.HASH_PATH_PREFIX utils.HASH_PATH_PREFIX = '' def fake_http_connect(status): class FakeConn(object): def __init__(self, status): self.status = status def getresponse(self): return self def read(self): return '' return lambda *args: FakeConn(status) orig_http_connect = object_server.http_connect try: for status in (200, 299): object_server.http_connect = fake_http_connect(status) self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': str(status)}, 'sda1') self.assertFalse( os.path.exists(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix def test_async_update_saves_on_timeout(self): _prefix = utils.HASH_PATH_PREFIX utils.HASH_PATH_PREFIX = '' def fake_http_connect(): class FakeConn(object): def getresponse(self): return sleep(1) return lambda *args: FakeConn() orig_http_connect = object_server.http_connect try: for status in (200, 299): object_server.http_connect = fake_http_connect() self.object_controller.node_timeout = 0.001 self.object_controller.async_update( 'PUT', 'a', 'c', 'o', '127.0.0.1:1234', 1, 'sdc1', {'x-timestamp': '1', 'x-out': str(status)}, 'sda1') self.assertTrue( os.path.exists(os.path.join( self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))) finally: object_server.http_connect = orig_http_connect utils.HASH_PATH_PREFIX = _prefix def test_container_update_no_async_update(self): given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.container_update( 'PUT', 'a', 'c', 'o', req, { 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1') self.assertEquals(given_args, []) def test_container_update(self): given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '123', 'X-Container-Host': 'chost', 'X-Container-Partition': 'cpartition', 'X-Container-Device': 'cdevice'}) self.object_controller.container_update( 'PUT', 'a', 'c', 'o', req, { 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1') self.assertEquals( given_args, [ 'PUT', 'a', 'c', 'o', 'chost', 'cpartition', 'cdevice', { 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}, 'sda1']) def test_container_update_bad_args(self): given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '123', 'X-Container-Host': 'chost,badhost', 'X-Container-Partition': 'cpartition', 'X-Container-Device': 'cdevice'}) self.object_controller.container_update( 'PUT', 'a', 'c', 'o', req, { 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1') self.assertEqual(given_args, []) def test_delete_at_update_on_put(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '123'}) self.object_controller.delete_at_update( 'DELETE', 2, 'a', 'c', 'o', req, 'sda1') self.assertEquals( given_args, [ 'DELETE', '.expiring_objects', '0000000000', '0000000002-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_negative(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. # Test negative is reset to 0 given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( 'DELETE', -2, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ 'DELETE', '.expiring_objects', '0000000000', '0000000000-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_cap(self): # Test how delete_at_update works when issued a delete for old # expiration info after a new put with no new expiration info. # Test past cap is reset to cap given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( 'DELETE', 12345678901, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ 'DELETE', '.expiring_objects', '9999936000', '9999999999-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_update_put_with_info(self): # Keep next test, # test_delete_at_update_put_with_info_but_missing_container, in sync # with this one but just missing the X-Delete-At-Container header. given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234', 'X-Delete-At-Container': '0', 'X-Delete-At-Host': '127.0.0.1:1234', 'X-Delete-At-Partition': '3', 'X-Delete-At-Device': 'sdc1'}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', req, 'sda1') self.assertEquals( given_args, [ 'PUT', '.expiring_objects', '0000000000', '0000000002-a/c/o', '127.0.0.1:1234', '3', 'sdc1', HeaderKeyDict({ 'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_update_put_with_info_but_missing_container(self): # Same as previous test, test_delete_at_update_put_with_info, but just # missing the X-Delete-At-Container header. given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update self.object_controller.logger = FakeLogger() req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234', 'X-Delete-At-Host': '127.0.0.1:1234', 'X-Delete-At-Partition': '3', 'X-Delete-At-Device': 'sdc1'}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', req, 'sda1') self.assertEquals( self.object_controller.logger.log_dict['warning'], [(('X-Delete-At-Container header must be specified for expiring ' 'objects background PUT to work properly. Making best guess as ' 'to the container name for now.',), {})]) def test_delete_at_update_delete(self): given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o', req, 'sda1') self.assertEquals( given_args, [ 'DELETE', '.expiring_objects', '0000000000', '0000000002-a/c/o', None, None, None, HeaderKeyDict({ 'x-timestamp': '1', 'x-trans-id': '1234', 'referer': 'DELETE http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_backend_replication(self): # If X-Backend-Replication: True delete_at_update should completely # short-circuit. given_args = [] def fake_async_update(*args): given_args.extend(args) self.object_controller.async_update = fake_async_update req = Request.blank( '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': 1, 'X-Trans-Id': '1234', 'X-Backend-Replication': 'True'}) self.object_controller.delete_at_update( 'DELETE', -2, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, []) def test_POST_calls_delete_at(self): given_args = [] def fake_delete_at_update(*args): given_args.extend(args) self.object_controller.delete_at_update = fake_delete_at_update req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, []) sleep(.00001) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'application/x-test'}) resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) self.assertEquals(given_args, []) sleep(.00001) timestamp1 = normalize_timestamp(time()) delete_at_timestamp1 = str(int(time() + 1000)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp1, 'Content-Type': 'application/x-test', 'X-Delete-At': delete_at_timestamp1}) resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) while given_args: given_args.pop() sleep(.00001) timestamp2 = normalize_timestamp(time()) delete_at_timestamp2 = str(int(time() + 2000)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp2, 'Content-Type': 'application/x-test', 'X-Delete-At': delete_at_timestamp2}) resp = self.object_controller.POST(req) self.assertEquals(resp.status_int, 202) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', req, 'sda1', 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) def test_PUT_calls_delete_at(self): given_args = [] def fake_delete_at_update(*args): given_args.extend(args) self.object_controller.delete_at_update = fake_delete_at_update req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, []) sleep(.00001) timestamp1 = normalize_timestamp(time()) delete_at_timestamp1 = str(int(time() + 1000)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp1, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', 'X-Delete-At': delete_at_timestamp1}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) while given_args: given_args.pop() sleep(.00001) timestamp2 = normalize_timestamp(time()) delete_at_timestamp2 = str(int(time() + 2000)) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp2, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', 'X-Delete-At': delete_at_timestamp2}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals( given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', req, 'sda1', 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) def test_GET_but_expired(self): test_time = time() + 10000 delete_at_timestamp = int(test_time + 100) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': normalize_timestamp(test_time)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) orig_time = object_server.time.time try: t = time() object_server.time.time = lambda: t delete_at_timestamp = int(t + 1) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': normalize_timestamp(test_time)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) finally: object_server.time.time = orig_time orig_time = object_server.time.time try: t = time() + 2 object_server.time.time = lambda: t req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': normalize_timestamp(t)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) finally: object_server.time.time = orig_time def test_HEAD_but_expired(self): test_time = time() + 10000 delete_at_timestamp = int(test_time + 100) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'X-Timestamp': normalize_timestamp(test_time)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) orig_time = object_server.time.time try: t = time() delete_at_timestamp = int(t + 1) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) object_server.time.time = lambda: t req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'X-Timestamp': normalize_timestamp(test_time)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) finally: object_server.time.time = orig_time orig_time = object_server.time.time try: t = time() + 2 object_server.time.time = lambda: t req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}, headers={'X-Timestamp': normalize_timestamp(time())}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) finally: object_server.time.time = orig_time def test_POST_but_expired(self): test_time = time() + 10000 delete_at_timestamp = int(test_time + 100) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1500)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 202) delete_at_timestamp = int(time() + 1) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 1000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) orig_time = object_server.time.time try: t = time() + 2 object_server.time.time = lambda: t req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(time())}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) finally: object_server.time.time = orig_time def test_DELETE_but_expired(self): test_time = time() + 10000 delete_at_timestamp = int(test_time + 100) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 2000), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) orig_time = object_server.time.time try: t = test_time + 100 object_server.time.time = lambda: float(t) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(time())}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) finally: object_server.time.time = orig_time def test_DELETE_if_delete_at_expired_still_deletes(self): test_time = time() + 10 test_timestamp = normalize_timestamp(test_time) delete_at_time = int(test_time + 10) delete_at_timestamp = str(delete_at_time) delete_at_container = str( delete_at_time / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': test_timestamp, 'X-Delete-At': delete_at_timestamp, 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) # sanity req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': test_timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body, 'TEST') objfile = os.path.join( self.testdir, 'sda1', storage_directory(diskfile.DATADIR, 'p', hash_path('a', 'c', 'o')), test_timestamp + '.data') self.assert_(os.path.isfile(objfile)) # move time past expirery with mock.patch('swift.obj.diskfile.time') as mock_time: mock_time.time.return_value = test_time + 100 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Timestamp': test_timestamp}) resp = req.get_response(self.object_controller) # request will 404 self.assertEquals(resp.status_int, 404) # but file still exists self.assert_(os.path.isfile(objfile)) # make the x-if-delete-at with all the right bits req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': delete_at_timestamp, 'X-If-Delete-At': delete_at_timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 404) self.assertFalse(os.path.isfile(objfile)) def test_DELETE_if_delete_at(self): test_time = time() + 10000 req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 99), 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 98)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) delete_at_timestamp = int(test_time - 1) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 97), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 95), 'X-If-Delete-At': str(int(test_time))}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 95)}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) delete_at_timestamp = int(test_time - 1) delete_at_container = str( delete_at_timestamp / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(test_time - 94), 'X-Delete-At': str(delete_at_timestamp), 'X-Delete-At-Container': delete_at_container, 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 92), 'X-If-Delete-At': str(int(test_time))}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 412) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 92), 'X-If-Delete-At': delete_at_timestamp}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 204) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': normalize_timestamp(test_time - 92), 'X-If-Delete-At': 'abc'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) def test_DELETE_calls_delete_at(self): given_args = [] def fake_delete_at_update(*args): given_args.extend(args) self.object_controller.delete_at_update = fake_delete_at_update timestamp1 = normalize_timestamp(time()) delete_at_timestamp1 = int(time() + 1000) delete_at_container1 = str( delete_at_timestamp1 / self.object_controller.expiring_objects_container_divisor * self.object_controller.expiring_objects_container_divisor) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp1, 'Content-Length': '4', 'Content-Type': 'application/octet-stream', 'X-Delete-At': str(delete_at_timestamp1), 'X-Delete-At-Container': delete_at_container1}) req.body = 'TEST' resp = self.object_controller.PUT(req) self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) while given_args: given_args.pop() sleep(.00001) timestamp2 = normalize_timestamp(time()) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}, headers={'X-Timestamp': timestamp2, 'Content-Type': 'application/octet-stream'}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 204) self.assertEquals(given_args, [ 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', req, 'sda1']) def test_PUT_delete_at_in_past(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'X-Delete-At': str(int(time() - 1)), 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) self.assertTrue('X-Delete-At in past' in resp.body) def test_POST_delete_at_in_past(self): req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'application/octet-stream'}) req.body = 'TEST' resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 201) req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': normalize_timestamp(time() + 1), 'X-Delete-At': str(int(time() - 1))}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 400) self.assertTrue('X-Delete-At in past' in resp.body) def test_REPLICATE_works(self): def fake_get_hashes(*args, **kwargs): return 0, {1: 2} def my_tpool_execute(func, *args, **kwargs): return func(*args, **kwargs) was_get_hashes = diskfile.get_hashes was_tpool_exe = tpool.execute try: diskfile.get_hashes = fake_get_hashes tpool.execute = my_tpool_execute req = Request.blank('/sda1/p/suff', environ={'REQUEST_METHOD': 'REPLICATE'}, headers={}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 200) p_data = pickle.loads(resp.body) self.assertEquals(p_data, {1: 2}) finally: tpool.execute = was_tpool_exe diskfile.get_hashes = was_get_hashes def test_REPLICATE_timeout(self): def fake_get_hashes(*args, **kwargs): raise Timeout() def my_tpool_execute(func, *args, **kwargs): return func(*args, **kwargs) was_get_hashes = diskfile.get_hashes was_tpool_exe = tpool.execute try: diskfile.get_hashes = fake_get_hashes tpool.execute = my_tpool_execute req = Request.blank('/sda1/p/suff', environ={'REQUEST_METHOD': 'REPLICATE'}, headers={}) self.assertRaises(Timeout, self.object_controller.REPLICATE, req) finally: tpool.execute = was_tpool_exe diskfile.get_hashes = was_get_hashes def test_REPLICATE_insufficient_storage(self): conf = {'devices': self.testdir, 'mount_check': 'true'} self.object_controller = object_server.ObjectController( conf, logger=debug_logger()) self.object_controller.bytes_per_sync = 1 def fake_check_mount(*args, **kwargs): return False with mock.patch("swift.obj.diskfile.check_mount", fake_check_mount): req = Request.blank('/sda1/p/suff', environ={'REQUEST_METHOD': 'REPLICATE'}, headers={}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 507) def test_REPLICATION_can_be_called(self): req = Request.blank('/sda1/p/other/suff', environ={'REQUEST_METHOD': 'REPLICATION'}, headers={}) resp = req.get_response(self.object_controller) self.assertEqual(resp.status_int, 200) def test_PUT_with_full_drive(self): class IgnoredBody(object): def __init__(self): self.read_called = False def read(self, size=-1): if not self.read_called: self.read_called = True return 'VERIFY' return '' def fake_fallocate(fd, size): raise OSError(42, 'Unable to fallocate(%d)' % size) orig_fallocate = diskfile.fallocate try: diskfile.fallocate = fake_fallocate timestamp = normalize_timestamp(time()) body_reader = IgnoredBody() req = Request.blank( '/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': body_reader}, headers={'X-Timestamp': timestamp, 'Content-Length': '6', 'Content-Type': 'application/octet-stream', 'Expect': '100-continue'}) resp = req.get_response(self.object_controller) self.assertEquals(resp.status_int, 507) self.assertFalse(body_reader.read_called) finally: diskfile.fallocate = orig_fallocate def test_global_conf_callback_does_nothing(self): preloaded_app_conf = {} global_conf = {} object_server.global_conf_callback(preloaded_app_conf, global_conf) self.assertEqual(preloaded_app_conf, {}) self.assertEqual(global_conf.keys(), ['replication_semaphore']) try: value = global_conf['replication_semaphore'][0].get_value() except NotImplementedError: # On some operating systems (at a minimum, OS X) it's not possible # to introspect the value of a semaphore raise SkipTest else: self.assertEqual(value, 4) def test_global_conf_callback_replication_semaphore(self): preloaded_app_conf = {'replication_concurrency': 123} global_conf = {} with mock.patch.object( object_server.multiprocessing, 'BoundedSemaphore', return_value='test1') as mocked_Semaphore: object_server.global_conf_callback(preloaded_app_conf, global_conf) self.assertEqual(preloaded_app_conf, {'replication_concurrency': 123}) self.assertEqual(global_conf, {'replication_semaphore': ['test1']}) mocked_Semaphore.assert_called_once_with(123) def test_handling_of_replication_semaphore_config(self): conf = {'devices': self.testdir, 'mount_check': 'false'} objsrv = object_server.ObjectController(conf) self.assertTrue(objsrv.replication_semaphore is None) conf['replication_semaphore'] = ['sema'] objsrv = object_server.ObjectController(conf) self.assertEqual(objsrv.replication_semaphore, 'sema') def test_serv_reserv(self): # Test replication_server flag was set from configuration file. conf = {'devices': self.testdir, 'mount_check': 'false'} self.assertEquals( object_server.ObjectController(conf).replication_server, None) for val in [True, '1', 'True', 'true']: conf['replication_server'] = val self.assertTrue( object_server.ObjectController(conf).replication_server) for val in [False, 0, '0', 'False', 'false', 'test_string']: conf['replication_server'] = val self.assertFalse( object_server.ObjectController(conf).replication_server) def test_list_allowed_methods(self): # Test list of allowed_methods obj_methods = ['DELETE', 'PUT', 'HEAD', 'GET', 'POST'] repl_methods = ['REPLICATE', 'REPLICATION'] for method_name in obj_methods: method = getattr(self.object_controller, method_name) self.assertFalse(hasattr(method, 'replication')) for method_name in repl_methods: method = getattr(self.object_controller, method_name) self.assertEquals(method.replication, True) def test_correct_allowed_method(self): # Test correct work for allowed method using # swift.obj.server.ObjectController.__call__ inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.app_factory( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false'}) def start_response(*args): # Sends args to outbuf outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} method_res = mock.MagicMock() mock_method = public(lambda x: mock.MagicMock(return_value=method_res)) with mock.patch.object(self.object_controller, method, new=mock_method): response = self.object_controller.__call__(env, start_response) self.assertEqual(response, method_res) def test_not_allowed_method(self): # Test correct work for NOT allowed method using # swift.obj.server.ObjectController.__call__ inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false'}, logger=FakeLogger()) def start_response(*args): # Sends args to outbuf outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} answer = ['

Method Not Allowed

The method is not ' 'allowed for this resource.

'] mock_method = replication(public(lambda x: mock.MagicMock())) with mock.patch.object(self.object_controller, method, new=mock_method): mock_method.replication = True with mock.patch('time.gmtime', mock.MagicMock(side_effect=[gmtime(10001.0)])): with mock.patch('time.time', mock.MagicMock(side_effect=[10000.0, 10001.0])): response = self.object_controller.__call__( env, start_response) self.assertEqual(response, answer) self.assertEqual( self.object_controller.logger.log_dict['info'], [(('None - - [01/Jan/1970:02:46:41 +0000] "PUT' ' /sda1/p/a/c/o" 405 - "-" "-" "-" 1.0000',), {})]) def test_not_utf8_and_not_logging_requests(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false', 'log_requests': 'false'}, logger=FakeLogger()) def start_response(*args): # Sends args to outbuf outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/\x00%20/%', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} answer = ['Invalid UTF8 or contains NULL'] mock_method = public(lambda x: mock.MagicMock()) with mock.patch.object(self.object_controller, method, new=mock_method): response = self.object_controller.__call__(env, start_response) self.assertEqual(response, answer) self.assertEqual(self.object_controller.logger.log_dict['info'], []) def test__call__returns_500(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false', 'log_requests': 'false'}, logger=FakeLogger()) def start_response(*args): # Sends args to outbuf outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} @public def mock_put_method(*args, **kwargs): raise Exception() with mock.patch.object(self.object_controller, method, new=mock_put_method): response = self.object_controller.__call__(env, start_response) self.assertTrue(response[0].startswith( 'Traceback (most recent call last):')) self.assertEqual( self.object_controller.logger.log_dict['exception'], [(('ERROR __call__ error with %(method)s %(path)s ', {'method': 'PUT', 'path': '/sda1/p/a/c/o'}), {}, '')]) self.assertEqual(self.object_controller.logger.log_dict['INFO'], []) def test_PUT_slow(self): inbuf = StringIO() errbuf = StringIO() outbuf = StringIO() self.object_controller = object_server.ObjectController( {'devices': self.testdir, 'mount_check': 'false', 'replication_server': 'false', 'log_requests': 'false', 'slow': '10'}, logger=FakeLogger()) def start_response(*args): # Sends args to outbuf outbuf.writelines(args) method = 'PUT' env = {'REQUEST_METHOD': method, 'SCRIPT_NAME': '', 'PATH_INFO': '/sda1/p/a/c/o', 'SERVER_NAME': '127.0.0.1', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.0', 'CONTENT_LENGTH': '0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', 'wsgi.input': inbuf, 'wsgi.errors': errbuf, 'wsgi.multithread': False, 'wsgi.multiprocess': False, 'wsgi.run_once': False} mock_method = public(lambda x: mock.MagicMock()) with mock.patch.object(self.object_controller, method, new=mock_method): with mock.patch('time.time', mock.MagicMock(side_effect=[10000.0, 10001.0])): with mock.patch('swift.obj.server.sleep', mock.MagicMock()) as ms: self.object_controller.__call__(env, start_response) ms.assert_called_with(9) self.assertEqual( self.object_controller.logger.log_dict['info'], []) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_replicator.py0000664000175400017540000007114112323703611022356 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import os import mock from gzip import GzipFile from shutil import rmtree import cPickle as pickle import time import tempfile from contextlib import contextmanager, closing from eventlet.green import subprocess from eventlet import Timeout, tpool from test.unit import FakeLogger from swift.common import utils from swift.common.utils import hash_path, mkdirs, normalize_timestamp from swift.common import ring from swift.obj import diskfile, replicator as object_replicator def _ips(): return ['127.0.0.0'] object_replicator.whataremyips = _ips def mock_http_connect(status): class FakeConn(object): def __init__(self, status, *args, **kwargs): self.status = status self.reason = 'Fake' self.host = args[0] self.port = args[1] self.method = args[4] self.path = args[5] self.with_exc = False self.headers = kwargs.get('headers', {}) def getresponse(self): if self.with_exc: raise Exception('test') return self def getheader(self, header): return self.headers[header] def read(self, amt=None): return pickle.dumps({}) def close(self): return return lambda *args, **kwargs: FakeConn(status, *args, **kwargs) process_errors = [] class MockProcess(object): ret_code = None ret_log = None check_args = None class Stream(object): def read(self): return MockProcess.ret_log.next() def __init__(self, *args, **kwargs): targs = MockProcess.check_args.next() for targ in targs: if targ not in args[0]: process_errors.append("Invalid: %s not in %s" % (targ, args)) self.stdout = self.Stream() def wait(self): return self.ret_code.next() @contextmanager def _mock_process(ret): orig_process = subprocess.Popen MockProcess.ret_code = (i[0] for i in ret) MockProcess.ret_log = (i[1] for i in ret) MockProcess.check_args = (i[2] for i in ret) object_replicator.subprocess.Popen = MockProcess yield object_replicator.subprocess.Popen = orig_process def _create_test_ring(path): testgz = os.path.join(path, 'object.ring.gz') intended_replica2part2dev_id = [ [0, 1, 2, 3, 4, 5, 6], [1, 2, 3, 0, 5, 6, 4], [2, 3, 0, 1, 6, 4, 5], ] intended_devs = [ {'id': 0, 'device': 'sda', 'zone': 0, 'ip': '127.0.0.0', 'port': 6000}, {'id': 1, 'device': 'sda', 'zone': 1, 'ip': '127.0.0.1', 'port': 6000}, {'id': 2, 'device': 'sda', 'zone': 2, 'ip': '127.0.0.2', 'port': 6000}, {'id': 3, 'device': 'sda', 'zone': 4, 'ip': '127.0.0.3', 'port': 6000}, {'id': 4, 'device': 'sda', 'zone': 5, 'ip': '127.0.0.4', 'port': 6000}, {'id': 5, 'device': 'sda', 'zone': 6, 'ip': 'fe80::202:b3ff:fe1e:8329', 'port': 6000}, {'id': 6, 'device': 'sda', 'zone': 7, 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', 'port': 6000}, ] intended_part_shift = 30 intended_reload_time = 15 with closing(GzipFile(testgz, 'wb')) as f: pickle.dump( ring.RingData(intended_replica2part2dev_id, intended_devs, intended_part_shift), f) return ring.Ring(path, ring_name='object', reload_time=intended_reload_time) class TestObjectReplicator(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) os.mkdir(self.devices) os.mkdir(os.path.join(self.devices, 'sda')) self.objects = os.path.join(self.devices, 'sda', 'objects') os.mkdir(self.objects) self.parts = {} for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) os.mkdir(os.path.join(self.objects, part)) self.ring = _create_test_ring(self.testdir) self.conf = dict( swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1') self.replicator = object_replicator.ObjectReplicator(self.conf) self.replicator.logger = FakeLogger() self.df_mgr = diskfile.DiskFileManager(self.conf, self.replicator.logger) def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_run_once(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect object_replicator.http_connect = mock_http_connect(200) cur_part = '0' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) object_replicator.http_connect = was_connector def test_check_ring(self): self.assertTrue(self.replicator.check_ring()) orig_check = self.replicator.next_check self.replicator.next_check = orig_check - 30 self.assertTrue(self.replicator.check_ring()) self.replicator.next_check = orig_check orig_ring_time = self.replicator.object_ring._mtime self.replicator.object_ring._mtime = orig_ring_time - 30 self.assertTrue(self.replicator.check_ring()) self.replicator.next_check = orig_check - 30 self.assertFalse(self.replicator.check_ring()) def test_collect_jobs_mkdirs_error(self): def blowup_mkdirs(path): raise OSError('Ow!') mkdirs_orig = object_replicator.mkdirs try: rmtree(self.objects, ignore_errors=1) object_replicator.mkdirs = blowup_mkdirs self.replicator.collect_jobs() self.assertTrue('exception' in self.replicator.logger.log_dict) self.assertEquals( len(self.replicator.logger.log_dict['exception']), 1) exc_args, exc_kwargs, exc_str = \ self.replicator.logger.log_dict['exception'][0] self.assertEquals(len(exc_args), 1) self.assertTrue(exc_args[0].startswith('ERROR creating ')) self.assertEquals(exc_kwargs, {}) self.assertEquals(exc_str, 'Ow!') finally: object_replicator.mkdirs = mkdirs_orig def test_collect_jobs(self): jobs = self.replicator.collect_jobs() jobs_to_delete = [j for j in jobs if j['delete']] jobs_by_part = {} for job in jobs: jobs_by_part[job['partition']] = job self.assertEquals(len(jobs_to_delete), 1) self.assertEquals('1', jobs_to_delete[0]['partition']) self.assertEquals( [node['id'] for node in jobs_by_part['0']['nodes']], [1, 2]) self.assertEquals( [node['id'] for node in jobs_by_part['1']['nodes']], [1, 2, 3]) self.assertEquals( [node['id'] for node in jobs_by_part['2']['nodes']], [2, 3]) self.assertEquals( [node['id'] for node in jobs_by_part['3']['nodes']], [3, 1]) for part in ['0', '1', '2', '3']: for node in jobs_by_part[part]['nodes']: self.assertEquals(node['device'], 'sda') self.assertEquals(jobs_by_part[part]['path'], os.path.join(self.objects, part)) def test_collect_jobs_handoffs_first(self): self.replicator.handoffs_first = True jobs = self.replicator.collect_jobs() self.assertTrue(jobs[0]['delete']) self.assertEquals('1', jobs[0]['partition']) def test_collect_jobs_removes_zbf(self): """ After running xfs_repair, a partition directory could become a zero-byte file. If this happens, collect_jobs() should clean it up and *not* create a job which will hit an exception as it tries to listdir() a file. """ # Surprise! Partition dir 1 is actually a zero-byte-file part_1_path = os.path.join(self.objects, '1') rmtree(part_1_path) with open(part_1_path, 'w'): pass self.assertTrue(os.path.isfile(part_1_path)) # sanity check jobs = self.replicator.collect_jobs() jobs_to_delete = [j for j in jobs if j['delete']] jobs_by_part = {} for job in jobs: jobs_by_part[job['partition']] = job self.assertEquals(len(jobs_to_delete), 0) self.assertEquals( [node['id'] for node in jobs_by_part['0']['nodes']], [1, 2]) self.assertFalse('1' in jobs_by_part) self.assertEquals( [node['id'] for node in jobs_by_part['2']['nodes']], [2, 3]) self.assertEquals( [node['id'] for node in jobs_by_part['3']['nodes']], [3, 1]) for part in ['0', '2', '3']: for node in jobs_by_part[part]['nodes']: self.assertEquals(node['device'], 'sda') self.assertEquals(jobs_by_part[part]['path'], os.path.join(self.objects, part)) self.assertFalse(os.path.exists(part_1_path)) self.assertEquals( [(('Removing partition directory which was a file: %s', part_1_path), {})], self.replicator.logger.log_dict['warning']) def test_delete_partition(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() self.assertFalse(os.access(part_path, os.F_OK)) def test_delete_partition_with_failures(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i == 0: # force one of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() # The path should still exist self.assertTrue(os.access(part_path, os.F_OK)) def test_delete_partition_with_handoff_delete(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i == 0: # force one of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() self.assertFalse(os.access(part_path, os.F_OK)) def test_delete_partition_with_handoff_delete_failures(self): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.handoff_delete = 2 df = self.df_mgr.get_diskfile('sda', '1', 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '1', data_dir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) nodes = [node for node in self.ring.get_part_nodes(1) if node['ip'] not in _ips()] process_arg_checker = [] for i, node in enumerate(nodes): rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], 1) if i in (0, 1): # force two of the rsync calls to fail ret_code = 1 else: ret_code = 0 process_arg_checker.append( (ret_code, '', ['rsync', whole_path_from, rsync_mod])) with _mock_process(process_arg_checker): self.replicator.replicate() # The file should still exist self.assertTrue(os.access(part_path, os.F_OK)) def test_delete_partition_override_params(self): df = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') mkdirs(df._datadir) part_path = os.path.join(self.objects, '1') self.assertTrue(os.access(part_path, os.F_OK)) self.replicator.replicate(override_devices=['sdb']) self.assertTrue(os.access(part_path, os.F_OK)) self.replicator.replicate(override_partitions=['9']) self.assertTrue(os.access(part_path, os.F_OK)) self.replicator.replicate(override_devices=['sda'], override_partitions=['1']) self.assertFalse(os.access(part_path, os.F_OK)) def test_run_once_recover_from_failure(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect try: object_replicator.http_connect = mock_http_connect(200) # Write some files into '1' and run replicate- they should be moved # to the other partitoins and then node should get deleted. cur_part = '1' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue(os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) for i, result in [('0', True), ('1', False), ('2', True), ('3', True)]: self.assertEquals(os.access( os.path.join(self.objects, i, diskfile.HASH_FILE), os.F_OK), result) finally: object_replicator.http_connect = was_connector def test_run_once_recover_from_timeout(self): replicator = object_replicator.ObjectReplicator( dict(swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1')) was_connector = object_replicator.http_connect was_get_hashes = object_replicator.get_hashes was_execute = tpool.execute self.get_hash_count = 0 try: def fake_get_hashes(*args, **kwargs): self.get_hash_count += 1 if self.get_hash_count == 3: # raise timeout on last call to get hashes raise Timeout() return 2, {'abc': 'def'} def fake_exc(tester, *args, **kwargs): if 'Error syncing partition' in args[0]: tester.i_failed = True self.i_failed = False object_replicator.http_connect = mock_http_connect(200) object_replicator.get_hashes = fake_get_hashes replicator.logger.exception = \ lambda *args, **kwargs: fake_exc(self, *args, **kwargs) # Write some files into '1' and run replicate- they should be moved # to the other partitions and then node should get deleted. cur_part = '1' df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o') mkdirs(df._datadir) f = open(os.path.join(df._datadir, normalize_timestamp(time.time()) + '.data'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, cur_part, data_dir) process_arg_checker = [] nodes = [node for node in self.ring.get_part_nodes(int(cur_part)) if node['ip'] not in _ips()] for node in nodes: rsync_mod = '%s::object/sda/objects/%s' % (node['ip'], cur_part) process_arg_checker.append( (0, '', ['rsync', whole_path_from, rsync_mod])) self.assertTrue(os.access(os.path.join(self.objects, '1', data_dir, ohash), os.F_OK)) with _mock_process(process_arg_checker): replicator.run_once() self.assertFalse(process_errors) self.assertFalse(self.i_failed) finally: object_replicator.http_connect = was_connector object_replicator.get_hashes = was_get_hashes tpool.execute = was_execute def test_run(self): with _mock_process([(0, '')] * 100): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.replicate() def test_run_withlog(self): with _mock_process([(0, "stuff in log")] * 100): with mock.patch('swift.obj.replicator.http_connect', mock_http_connect(200)): self.replicator.replicate() def test_sync_just_calls_sync_method(self): self.replicator.sync_method = mock.MagicMock() self.replicator.sync('node', 'job', 'suffixes') self.replicator.sync_method.assert_called_once_with( 'node', 'job', 'suffixes') @mock.patch('swift.obj.replicator.tpool_reraise', autospec=True) @mock.patch('swift.obj.replicator.http_connect', autospec=True) def test_update(self, mock_http, mock_tpool_reraise): def set_default(self): self.replicator.suffix_count = 0 self.replicator.suffix_sync = 0 self.replicator.suffix_hash = 0 self.replicator.replication_count = 0 self.replicator.partition_times = [] self.headers = {'Content-Length': '0', 'user-agent': 'obj-replicator %s' % os.getpid()} self.replicator.logger = mock_logger = mock.MagicMock() mock_tpool_reraise.return_value = (0, {}) all_jobs = self.replicator.collect_jobs() jobs = [job for job in all_jobs if not job['delete']] mock_http.return_value = answer = mock.MagicMock() answer.getresponse.return_value = resp = mock.MagicMock() # Check uncorrect http_connect with status 507 and # count of attempts and call args resp.status = 507 error = '%(ip)s/%(device)s responded as unmounted' expect = 'Error syncing partition' for job in jobs: set_default(self) self.replicator.update(job) self.assertTrue(error in mock_logger.error.call_args[0][0]) self.assertTrue(expect in mock_logger.exception.call_args[0][0]) self.assertEquals(len(self.replicator.partition_times), 1) self.assertEquals(mock_http.call_count, len(self.ring._devs) - 1) reqs = [] for node in job['nodes']: reqs.append(mock.call(node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', '', headers=self.headers)) if job['partition'] == '0': self.assertEquals(self.replicator.suffix_hash, 0) mock_http.assert_has_calls(reqs, any_order=True) mock_http.reset_mock() mock_logger.reset_mock() # Check uncorrect http_connect with status 400 != HTTP_OK resp.status = 400 error = 'Invalid response %(resp)s from %(ip)s' for job in jobs: set_default(self) self.replicator.update(job) self.assertTrue(error in mock_logger.error.call_args[0][0]) self.assertEquals(len(self.replicator.partition_times), 1) mock_logger.reset_mock() # Check successful http_connection and exception with # uncorrect pickle.loads(resp.read()) resp.status = 200 expect = 'Error syncing with node:' for job in jobs: set_default(self) self.replicator.update(job) self.assertTrue(expect in mock_logger.exception.call_args[0][0]) self.assertEquals(len(self.replicator.partition_times), 1) mock_logger.reset_mock() # Check successful http_connection and correct # pickle.loads(resp.read()) for non local node resp.status = 200 local_job = None resp.read.return_value = pickle.dumps({}) for job in jobs: set_default(self) if job['partition'] == '0': local_job = job.copy() continue self.replicator.update(job) self.assertEquals(mock_logger.exception.call_count, 0) self.assertEquals(mock_logger.error.call_count, 0) self.assertEquals(len(self.replicator.partition_times), 1) self.assertEquals(self.replicator.suffix_hash, 0) self.assertEquals(self.replicator.suffix_sync, 0) self.assertEquals(self.replicator.suffix_count, 0) mock_logger.reset_mock() # Check successful http_connect and sync for local node mock_tpool_reraise.return_value = (1, {'a83': 'ba47fd314242ec8c' '7efb91f5d57336e4'}) resp.read.return_value = pickle.dumps({'a83': 'c130a2c17ed45102a' 'ada0f4eee69494ff'}) set_default(self) self.replicator.sync = fake_func = mock.MagicMock() self.replicator.update(local_job) reqs = [] for node in local_job['nodes']: reqs.append(mock.call(node, local_job, ['a83'])) fake_func.assert_has_calls(reqs, any_order=True) self.assertEquals(fake_func.call_count, 2) self.assertEquals(self.replicator.replication_count, 1) self.assertEquals(self.replicator.suffix_sync, 2) self.assertEquals(self.replicator.suffix_hash, 1) self.assertEquals(self.replicator.suffix_count, 1) mock_http.reset_mock() mock_logger.reset_mock() # test for replication params repl_job = local_job.copy() for node in repl_job['nodes']: node['replication_ip'] = '127.0.0.11' node['replication_port'] = '6011' set_default(self) self.replicator.update(repl_job) reqs = [] for node in repl_job['nodes']: reqs.append(mock.call(node['replication_ip'], node['replication_port'], node['device'], repl_job['partition'], 'REPLICATE', '', headers=self.headers)) reqs.append(mock.call(node['replication_ip'], node['replication_port'], node['device'], repl_job['partition'], 'REPLICATE', '/a83', headers=self.headers)) mock_http.assert_has_calls(reqs, any_order=True) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_auditor.py0000664000175400017540000004447212323703611021670 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from test import unit import unittest import mock import os import time import string from shutil import rmtree from hashlib import md5 from tempfile import mkdtemp from test.unit import FakeLogger from swift.obj import auditor from swift.obj.diskfile import DiskFile, write_metadata, invalidate_hash, \ DATADIR, DiskFileManager, AuditLocation from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ storage_directory class TestAuditor(unittest.TestCase): def setUp(self): self.testdir = os.path.join(mkdtemp(), 'tmp_test_object_auditor') self.devices = os.path.join(self.testdir, 'node') self.rcache = os.path.join(self.testdir, 'object.recon') self.logger = FakeLogger() rmtree(self.testdir, ignore_errors=1) mkdirs(os.path.join(self.devices, 'sda')) self.objects = os.path.join(self.devices, 'sda', 'objects') os.mkdir(os.path.join(self.devices, 'sdb')) self.objects_2 = os.path.join(self.devices, 'sdb', 'objects') os.mkdir(self.objects) self.parts = {} for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) os.mkdir(os.path.join(self.objects, part)) self.conf = dict( devices=self.devices, mount_check='false', object_size_stats='10,100,1024,10240') self.df_mgr = DiskFileManager(self.conf, self.logger) self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') def tearDown(self): rmtree(os.path.dirname(self.testdir), ignore_errors=1) unit.xattr_data = {} def test_object_audit_extra_data(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() timestamp = str(normalize_timestamp(time.time())) metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) pre_quarantines = auditor_worker.quarantines auditor_worker.object_audit( AuditLocation(self.disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines) os.write(writer._fd, 'extra_data') auditor_worker.object_audit( AuditLocation(self.disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_diff_data(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) data = '0' * 1024 etag = md5() timestamp = str(normalize_timestamp(time.time())) with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) pre_quarantines = auditor_worker.quarantines # remake so it will have metadata self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') auditor_worker.object_audit( AuditLocation(self.disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines) etag = md5() etag.update('1' + '0' * 1023) etag = etag.hexdigest() metadata['ETag'] = etag with self.disk_file.create() as writer: writer.write(data) writer.put(metadata) auditor_worker.object_audit( AuditLocation(self.disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_no_meta(self): timestamp = str(normalize_timestamp(time.time())) path = os.path.join(self.disk_file._datadir, timestamp + '.data') mkdirs(self.disk_file._datadir) fp = open(path, 'w') fp.write('0' * 1024) fp.close() invalidate_hash(os.path.dirname(self.disk_file._datadir)) auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) pre_quarantines = auditor_worker.quarantines auditor_worker.object_audit( AuditLocation(self.disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_audit_will_not_swallow_errors_in_tests(self): timestamp = str(normalize_timestamp(time.time())) path = os.path.join(self.disk_file._datadir, timestamp + '.data') mkdirs(self.disk_file._datadir) with open(path, 'w') as f: write_metadata(f, {'name': '/a/c/o'}) auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) def blowup(*args): raise NameError('tpyo') with mock.patch.object(DiskFileManager, 'get_diskfile_from_audit_location', blowup): self.assertRaises(NameError, auditor_worker.object_audit, AuditLocation(os.path.dirname(path), 'sda', '0')) def test_failsafe_object_audit_will_swallow_errors_in_tests(self): timestamp = str(normalize_timestamp(time.time())) path = os.path.join(self.disk_file._datadir, timestamp + '.data') mkdirs(self.disk_file._datadir) with open(path, 'w') as f: write_metadata(f, {'name': '/a/c/o'}) auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) def blowup(*args): raise NameError('tpyo') with mock.patch('swift.obj.diskfile.DiskFile', blowup): auditor_worker.failsafe_object_audit( AuditLocation(os.path.dirname(path), 'sda', '0')) self.assertEquals(auditor_worker.errors, 1) def test_generic_exception_handling(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) timestamp = str(normalize_timestamp(time.time())) pre_errors = auditor_worker.errors data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) with mock.patch('swift.obj.diskfile.DiskFile', lambda *_: 1 / 0): auditor_worker.audit_all_objects() self.assertEquals(auditor_worker.errors, pre_errors + 1) def test_object_run_once_pass(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) auditor_worker.log_time = 0 timestamp = str(normalize_timestamp(time.time())) pre_quarantines = auditor_worker.quarantines data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) auditor_worker.audit_all_objects() self.assertEquals(auditor_worker.quarantines, pre_quarantines) self.assertEquals(auditor_worker.stats_buckets[1024], 1) self.assertEquals(auditor_worker.stats_buckets[10240], 0) def test_object_run_once_no_sda(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) timestamp = str(normalize_timestamp(time.time())) pre_quarantines = auditor_worker.quarantines data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) os.write(writer._fd, 'extra_data') auditor_worker.audit_all_objects() self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_run_once_multi_devices(self): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) timestamp = str(normalize_timestamp(time.time())) pre_quarantines = auditor_worker.quarantines data = '0' * 10 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) auditor_worker.audit_all_objects() self.disk_file = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'ob') data = '1' * 10 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) os.write(writer._fd, 'extra_data') auditor_worker.audit_all_objects() self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1) def test_object_run_fast_track_non_zero(self): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 data = '0' * 1024 etag = md5() with self.disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) etag = md5() etag.update('1' + '0' * 1023) etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(writer._fd, metadata) quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') kwargs = {'mode': 'once'} kwargs['zero_byte_fps'] = 50 self.auditor.run_audit(**kwargs) self.assertFalse(os.path.isdir(quarantine_path)) del(kwargs['zero_byte_fps']) self.auditor.run_audit(**kwargs) self.assertTrue(os.path.isdir(quarantine_path)) def setup_bad_zero_byte(self, with_ts=False): self.auditor = auditor.ObjectAuditor(self.conf) self.auditor.log_time = 0 ts_file_path = '' if with_ts: name_hash = hash_path('a', 'c', 'o') dir_path = os.path.join( self.devices, 'sda', storage_directory(DATADIR, '0', name_hash)) ts_file_path = os.path.join(dir_path, '99999.ts') if not os.path.exists(dir_path): mkdirs(dir_path) fp = open(ts_file_path, 'w') write_metadata(fp, {'X-Timestamp': '99999', 'name': '/a/c/o'}) fp.close() etag = md5() with self.disk_file.create() as writer: etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': str(normalize_timestamp(time.time())), 'Content-Length': 10, } writer.put(metadata) etag = md5() etag = etag.hexdigest() metadata['ETag'] = etag write_metadata(writer._fd, metadata) return ts_file_path def test_object_run_fast_track_all(self): self.setup_bad_zero_byte() kwargs = {'mode': 'once'} self.auditor.run_audit(**kwargs) quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') self.assertTrue(os.path.isdir(quarantine_path)) def test_object_run_fast_track_zero(self): self.setup_bad_zero_byte() kwargs = {'mode': 'once'} kwargs['zero_byte_fps'] = 50 self.auditor.run_audit(**kwargs) quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') self.assertTrue(os.path.isdir(quarantine_path)) def test_object_run_fast_track_zero_check_closed(self): rat = [False] class FakeFile(DiskFile): def _quarantine(self, data_file, msg): rat[0] = True DiskFile._quarantine(self, data_file, msg) self.setup_bad_zero_byte() was_df = auditor.diskfile.DiskFile try: auditor.diskfile.DiskFile = FakeFile kwargs = {'mode': 'once'} kwargs['zero_byte_fps'] = 50 self.auditor.run_audit(**kwargs) quarantine_path = os.path.join(self.devices, 'sda', 'quarantined', 'objects') self.assertTrue(os.path.isdir(quarantine_path)) self.assertTrue(rat[0]) finally: auditor.diskfile.DiskFile = was_df def test_with_tombstone(self): ts_file_path = self.setup_bad_zero_byte(with_ts=True) self.assertTrue(ts_file_path.endswith('ts')) kwargs = {'mode': 'once'} self.auditor.run_audit(**kwargs) self.assertTrue(os.path.exists(ts_file_path)) def test_sleeper(self): auditor.SLEEP_BETWEEN_AUDITS = 0.10 my_auditor = auditor.ObjectAuditor(self.conf) start = time.time() my_auditor._sleep() delta_t = time.time() - start self.assert_(delta_t > 0.08) self.assert_(delta_t < 0.12) def test_run_audit(self): class StopForever(Exception): pass class ObjectAuditorMock(object): check_args = () check_kwargs = {} check_device_dir = None fork_called = 0 master = 0 wait_called = 0 def mock_run(self, *args, **kwargs): self.check_args = args self.check_kwargs = kwargs if 'zero_byte_fps' in kwargs: self.check_device_dir = kwargs.get('device_dirs') def mock_sleep(self): raise StopForever('stop') def mock_fork(self): self.fork_called += 1 if self.master: return self.fork_called else: return 0 def mock_wait(self): self.wait_called += 1 return (self.wait_called, 0) for i in string.ascii_letters[2:26]: mkdirs(os.path.join(self.devices, 'sd%s' % i)) my_auditor = auditor.ObjectAuditor(dict(devices=self.devices, mount_check='false', zero_byte_files_per_second=89)) mocker = ObjectAuditorMock() my_auditor.run_audit = mocker.mock_run my_auditor._sleep = mocker.mock_sleep was_fork = os.fork was_wait = os.wait try: os.fork = mocker.mock_fork os.wait = mocker.mock_wait self.assertRaises(StopForever, my_auditor.run_forever, zero_byte_fps=50) self.assertEquals(mocker.check_kwargs['zero_byte_fps'], 50) self.assertEquals(mocker.fork_called, 0) self.assertRaises(SystemExit, my_auditor.run_forever) self.assertEquals(mocker.fork_called, 1) self.assertEquals(mocker.check_kwargs['zero_byte_fps'], 89) self.assertEquals(mocker.check_device_dir, None) self.assertEquals(mocker.check_args, ()) device_list = ['sd%s' % i for i in string.ascii_letters[2:10]] device_string = ','.join(device_list) device_string_bogus = device_string + ',bogus' mocker.fork_called = 0 self.assertRaises(SystemExit, my_auditor.run_once, devices=device_string_bogus) self.assertEquals(mocker.fork_called, 1) self.assertEquals(mocker.check_kwargs['zero_byte_fps'], 89) self.assertEquals(sorted(mocker.check_device_dir), device_list) mocker.master = 1 mocker.fork_called = 0 self.assertRaises(StopForever, my_auditor.run_forever) # Fork is called 3 times since the zbf process is forked twice self.assertEquals(mocker.fork_called, 3) self.assertEquals(mocker.wait_called, 3) finally: os.fork = was_fork os.wait = was_wait if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_ssync_receiver.py0000664000175400017540000015571612323703611023250 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import contextlib import os import shutil import StringIO import tempfile import unittest import eventlet import mock from swift.common import constraints from swift.common import exceptions from swift.common import swob from swift.common import utils from swift.obj import diskfile from swift.obj import server from swift.obj import ssync_receiver from test import unit class TestReceiver(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' # Not sure why the test.unit stuff isn't taking effect here; so I'm # reenforcing it. diskfile.getxattr = unit._getxattr diskfile.setxattr = unit._setxattr self.testdir = os.path.join( tempfile.mkdtemp(), 'tmp_test_ssync_receiver') utils.mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) conf = { 'devices': self.testdir, 'mount_check': 'false', 'replication_one_per_device': 'false', 'log_requests': 'false'} self.controller = server.ObjectController(conf) self.controller.bytes_per_sync = 1 self.account1 = 'a' self.container1 = 'c' self.object1 = 'o1' self.name1 = '/' + '/'.join(( self.account1, self.container1, self.object1)) self.hash1 = utils.hash_path( self.account1, self.container1, self.object1) self.ts1 = '1372800001.00000' self.metadata1 = { 'name': self.name1, 'X-Timestamp': self.ts1, 'Content-Length': '0'} self.account2 = 'a' self.container2 = 'c' self.object2 = 'o2' self.name2 = '/' + '/'.join(( self.account2, self.container2, self.object2)) self.hash2 = utils.hash_path( self.account2, self.container2, self.object2) self.ts2 = '1372800002.00000' self.metadata2 = { 'name': self.name2, 'X-Timestamp': self.ts2, 'Content-Length': '0'} def tearDown(self): shutil.rmtree(os.path.dirname(self.testdir)) def body_lines(self, body): lines = [] for line in body.split('\n'): line = line.strip() if line: lines.append(line) return lines def test_REPLICATION_semaphore_locked(self): with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: self.controller.logger = mock.MagicMock() mocked_replication_semaphore.acquire.return_value = False req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 503 '

Service Unavailable

The " "server is currently unavailable. Please try again at a " "later time.

'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_REPLICATION_calls_replication_lock(self): with mock.patch.object( self.controller._diskfile_mgr, 'replication_lock') as \ mocked_replication_lock: req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) mocked_replication_lock.assert_called_once_with('sda1') def test_REPLICATION_replication_lock_fail(self): def _mock(path): with exceptions.ReplicationLockTimeout(0.01, '/somewhere/' + path): eventlet.sleep(0.05) with mock.patch.object( self.controller._diskfile_mgr, 'replication_lock', _mock): self.controller._diskfile_mgr self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 0 '0.01 seconds: /somewhere/sda1'"]) self.controller.logger.debug.assert_called_once_with( 'None/sda1/1 REPLICATION LOCK TIMEOUT: 0.01 seconds: ' '/somewhere/sda1') def test_REPLICATION_initial_path(self): with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( '/device', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 0 'Invalid path: /device'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( '/device/', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 0 'Invalid path: /device/'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':ERROR: 0 "Looking for :MISSING_CHECK: START got \'\'"']) self.assertEqual(resp.status_int, 200) mocked_replication_semaphore.acquire.assert_called_once_with(0) mocked_replication_semaphore.release.assert_called_once_with() with mock.patch.object( self.controller, 'replication_semaphore') as \ mocked_replication_semaphore: req = swob.Request.blank( '/device/partition/junk', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 0 'Invalid path: /device/partition/junk'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_replication_semaphore.acquire.called) self.assertFalse(mocked_replication_semaphore.release.called) def test_REPLICATION_mount_check(self): with contextlib.nested( mock.patch.object( self.controller, 'replication_semaphore'), mock.patch.object( self.controller._diskfile_mgr, 'mount_check', False), mock.patch.object( constraints, 'check_mount', return_value=False)) as ( mocked_replication_semaphore, mocked_mount_check, mocked_check_mount): req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':ERROR: 0 "Looking for :MISSING_CHECK: START got \'\'"']) self.assertEqual(resp.status_int, 200) self.assertFalse(mocked_check_mount.called) with contextlib.nested( mock.patch.object( self.controller, 'replication_semaphore'), mock.patch.object( self.controller._diskfile_mgr, 'mount_check', True), mock.patch.object( constraints, 'check_mount', return_value=False)) as ( mocked_replication_semaphore, mocked_mount_check, mocked_check_mount): req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 507 '

Insufficient Storage

There " "was not enough space to save the resource. Drive: " "device

'"]) self.assertEqual(resp.status_int, 200) mocked_check_mount.assert_called_once_with( self.controller._diskfile_mgr.devices, 'device') mocked_check_mount.reset_mock() mocked_check_mount.return_value = True req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':ERROR: 0 "Looking for :MISSING_CHECK: START got \'\'"']) self.assertEqual(resp.status_int, 200) mocked_check_mount.assert_called_once_with( self.controller._diskfile_mgr.devices, 'device') def test_REPLICATION_Exception(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def get_socket(self): return self.mock_socket with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\nBad content is here') req.remote_addr = '1.2.3.4' mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Got no headers for Bad content is here'"]) self.assertEqual(resp.status_int, 200) mock_shutdown_safe.assert_called_once_with( mock_wsgi_input.mock_socket) mock_wsgi_input.mock_socket.close.assert_called_once_with() self.controller.logger.exception.assert_called_once_with( '1.2.3.4/device/partition EXCEPTION in replication.Receiver') def test_REPLICATION_Exception_Exception(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def get_socket(self): return self.mock_socket with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\nBad content is here') req.remote_addr = mock.MagicMock() req.remote_addr.__str__ = mock.Mock( side_effect=Exception("can't stringify this")) mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END']) self.assertEqual(resp.status_int, 200) mock_shutdown_safe.assert_called_once_with( mock_wsgi_input.mock_socket) mock_wsgi_input.mock_socket.close.assert_called_once_with() self.controller.logger.exception.assert_called_once_with( 'EXCEPTION in replication.Receiver') def test_MISSING_CHECK_timeout(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def readline(self, sizehint=-1): line = StringIO.StringIO.readline(self) if line.startswith('hash'): eventlet.sleep(0.1) return line def get_socket(self): return self.mock_socket self.controller.client_timeout = 0.01 with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' 'hash ts\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') req.remote_addr = '2.3.4.5' mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 408 '0.01 seconds: missing_check line'"]) self.assertEqual(resp.status_int, 200) self.assertTrue(mock_shutdown_safe.called) self.controller.logger.error.assert_called_once_with( '2.3.4.5/sda1/1 TIMEOUT in replication.Receiver: ' '0.01 seconds: missing_check line') def test_MISSING_CHECK_other_exception(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def readline(self, sizehint=-1): line = StringIO.StringIO.readline(self) if line.startswith('hash'): raise Exception('test exception') return line def get_socket(self): return self.mock_socket self.controller.client_timeout = 0.01 with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' 'hash ts\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') req.remote_addr = '3.4.5.6' mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [":ERROR: 0 'test exception'"]) self.assertEqual(resp.status_int, 200) self.assertTrue(mock_shutdown_safe.called) self.controller.logger.exception.assert_called_once_with( '3.4.5.6/sda1/1 EXCEPTION in replication.Receiver') def test_MISSING_CHECK_empty_list(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_MISSING_CHECK_have_none(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', self.hash1, self.hash2, ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_MISSING_CHECK_have_one_exact(self): object_dir = utils.storage_directory( os.path.join(self.testdir, 'sda1', diskfile.DATADIR), '1', self.hash1) utils.mkdirs(object_dir) fp = open(os.path.join(object_dir, self.ts1 + '.data'), 'w+') fp.write('1') fp.flush() self.metadata1['Content-Length'] = '1' diskfile.write_metadata(fp, self.metadata1) self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', self.hash2, ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_MISSING_CHECK_have_one_newer(self): object_dir = utils.storage_directory( os.path.join(self.testdir, 'sda1', diskfile.DATADIR), '1', self.hash1) utils.mkdirs(object_dir) newer_ts1 = utils.normalize_timestamp(float(self.ts1) + 1) self.metadata1['X-Timestamp'] = newer_ts1 fp = open(os.path.join(object_dir, newer_ts1 + '.data'), 'w+') fp.write('1') fp.flush() self.metadata1['Content-Length'] = '1' diskfile.write_metadata(fp, self.metadata1) self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', self.hash2, ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_MISSING_CHECK_have_one_older(self): object_dir = utils.storage_directory( os.path.join(self.testdir, 'sda1', diskfile.DATADIR), '1', self.hash1) utils.mkdirs(object_dir) older_ts1 = utils.normalize_timestamp(float(self.ts1) - 1) self.metadata1['X-Timestamp'] = older_ts1 fp = open(os.path.join(object_dir, older_ts1 + '.data'), 'w+') fp.write('1') fp.flush() self.metadata1['Content-Length'] = '1' diskfile.write_metadata(fp, self.metadata1) self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/sda1/1', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n' + self.hash1 + ' ' + self.ts1 + '\r\n' + self.hash2 + ' ' + self.ts2 + '\r\n' ':MISSING_CHECK: END\r\n' ':UPDATES: START\r\n:UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', self.hash1, self.hash2, ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.error.called) self.assertFalse(self.controller.logger.exception.called) def test_UPDATES_timeout(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def readline(self, sizehint=-1): line = StringIO.StringIO.readline(self) if line.startswith('DELETE'): eventlet.sleep(0.1) return line def get_socket(self): return self.mock_socket self.controller.client_timeout = 0.01 with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n' ':UPDATES: END\r\n') req.remote_addr = '2.3.4.5' mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 408 '0.01 seconds: updates line'"]) self.assertEqual(resp.status_int, 200) mock_shutdown_safe.assert_called_once_with( mock_wsgi_input.mock_socket) mock_wsgi_input.mock_socket.close.assert_called_once_with() self.controller.logger.error.assert_called_once_with( '2.3.4.5/device/partition TIMEOUT in replication.Receiver: ' '0.01 seconds: updates line') def test_UPDATES_other_exception(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def readline(self, sizehint=-1): line = StringIO.StringIO.readline(self) if line.startswith('DELETE'): raise Exception('test exception') return line def get_socket(self): return self.mock_socket self.controller.client_timeout = 0.01 with mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe') as \ mock_shutdown_safe: self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n' ':UPDATES: END\r\n') req.remote_addr = '3.4.5.6' mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'test exception'"]) self.assertEqual(resp.status_int, 200) mock_shutdown_safe.assert_called_once_with( mock_wsgi_input.mock_socket) mock_wsgi_input.mock_socket.close.assert_called_once_with() self.controller.logger.exception.assert_called_once_with( '3.4.5.6/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_no_problems_no_hard_disconnect(self): class _Wrapper(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) self.mock_socket = mock.MagicMock() def get_socket(self): return self.mock_socket self.controller.client_timeout = 0.01 with contextlib.nested( mock.patch.object( ssync_receiver.eventlet.greenio, 'shutdown_safe'), mock.patch.object( self.controller, 'DELETE', return_value=swob.HTTPNoContent())) as ( mock_shutdown_safe, mock_delete): req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n' ':UPDATES: END\r\n') mock_wsgi_input = _Wrapper(req.body) req.environ['wsgi.input'] = mock_wsgi_input resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(mock_shutdown_safe.called) self.assertFalse(mock_wsgi_input.mock_socket.close.called) def test_UPDATES_bad_subrequest_line(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'bad_subrequest_line\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'need more than 1 value to unpack'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') with mock.patch.object( self.controller, 'DELETE', return_value=swob.HTTPNoContent()): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n' 'bad_subrequest_line2') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'need more than 1 value to unpack'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_no_headers(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Got no headers for DELETE /a/c/o'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_bad_headers(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'Bad-Header Test\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'need more than 1 value to unpack'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'Good-Header: Test\r\n' 'Bad-Header Test\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'need more than 1 value to unpack'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_bad_content_length(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' 'Content-Length: a\r\n\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':ERROR: 0 "invalid literal for int() with base 10: \'a\'"']) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_content_length_with_DELETE(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'Content-Length: 1\r\n\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'DELETE subrequest with content-length /a/c/o'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_no_content_length_with_PUT(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'No content-length sent for PUT /a/c/o'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_early_termination(self): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' 'Content-Length: 1\r\n\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Early termination for PUT /a/c/o'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') def test_UPDATES_failures(self): @server.public def _DELETE(request): if request.path == '/device/partition/a/c/works': return swob.HTTPOk() else: return swob.HTTPInternalServerError() # failures never hit threshold with mock.patch.object(self.controller, 'DELETE', _DELETE): self.controller.replication_failure_threshold = 4 self.controller.replication_failure_ratio = 1.5 self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 500 'ERROR: With :UPDATES: 3 failures to 0 " "successes'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) # failures hit threshold and no successes, so ratio is like infinity with mock.patch.object(self.controller, 'DELETE', _DELETE): self.controller.replication_failure_threshold = 4 self.controller.replication_failure_ratio = 1.5 self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' ':UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Too many 4 failures to 0 successes'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') self.assertFalse(self.controller.logger.error.called) # failures hit threshold and ratio hits 1.33333333333 with mock.patch.object(self.controller, 'DELETE', _DELETE): self.controller.replication_failure_threshold = 4 self.controller.replication_failure_ratio = 1.5 self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/works\r\n\r\n' 'DELETE /a/c/works\r\n\r\n' 'DELETE /a/c/works\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' ':UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 500 'ERROR: With :UPDATES: 4 failures to 3 " "successes'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) # failures hit threshold and ratio hits 2.0 with mock.patch.object(self.controller, 'DELETE', _DELETE): self.controller.replication_failure_threshold = 4 self.controller.replication_failure_ratio = 1.5 self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/works\r\n\r\n' 'DELETE /a/c/works\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' 'DELETE /a/c/o\r\n\r\n' ':UPDATES: END\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Too many 4 failures to 2 successes'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') self.assertFalse(self.controller.logger.error.called) def test_UPDATES_PUT(self): _PUT_request = [None] @server.public def _PUT(request): _PUT_request[0] = request request.read_body = request.environ['wsgi.input'].read() return swob.HTTPOk() with mock.patch.object(self.controller, 'PUT', _PUT): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o\r\n' 'Content-Length: 1\r\n' 'X-Timestamp: 1364456113.12344\r\n' 'X-Object-Meta-Test1: one\r\n' 'Content-Encoding: gzip\r\n' 'Specialty-Header: value\r\n' '\r\n' '1') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) req = _PUT_request[0] self.assertEqual(req.path, '/device/partition/a/c/o') self.assertEqual(req.content_length, 1) self.assertEqual(req.headers, { 'Content-Length': '1', 'X-Timestamp': '1364456113.12344', 'X-Object-Meta-Test1': 'one', 'Content-Encoding': 'gzip', 'Specialty-Header': 'value', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp x-object-meta-test1 ' 'content-encoding specialty-header')}) self.assertEqual(req.read_body, '1') def test_UPDATES_DELETE(self): _DELETE_request = [None] @server.public def _DELETE(request): _DELETE_request[0] = request return swob.HTTPOk() with mock.patch.object(self.controller, 'DELETE', _DELETE): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) req = _DELETE_request[0] self.assertEqual(req.path, '/device/partition/a/c/o') self.assertEqual(req.headers, { 'X-Timestamp': '1364456113.76334', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': 'x-timestamp'}) def test_UPDATES_BONK(self): _BONK_request = [None] @server.public def _BONK(request): _BONK_request[0] = request return swob.HTTPOk() self.controller.BONK = _BONK self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'BONK /a/c/o\r\n' 'X-Timestamp: 1364456113.76334\r\n' '\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 0 'Invalid subrequest method BONK'"]) self.assertEqual(resp.status_int, 200) self.controller.logger.exception.assert_called_once_with( 'None/device/partition EXCEPTION in replication.Receiver') self.assertEqual(_BONK_request[0], None) def test_UPDATES_multiple(self): _requests = [] @server.public def _PUT(request): _requests.append(request) request.read_body = request.environ['wsgi.input'].read() return swob.HTTPOk() @server.public def _DELETE(request): _requests.append(request) return swob.HTTPOk() with contextlib.nested( mock.patch.object(self.controller, 'PUT', _PUT), mock.patch.object(self.controller, 'DELETE', _DELETE)): self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o1\r\n' 'Content-Length: 1\r\n' 'X-Timestamp: 1364456113.00001\r\n' 'X-Object-Meta-Test1: one\r\n' 'Content-Encoding: gzip\r\n' 'Specialty-Header: value\r\n' '\r\n' '1' 'DELETE /a/c/o2\r\n' 'X-Timestamp: 1364456113.00002\r\n' '\r\n' 'PUT /a/c/o3\r\n' 'Content-Length: 3\r\n' 'X-Timestamp: 1364456113.00003\r\n' '\r\n' '123' 'PUT /a/c/o4\r\n' 'Content-Length: 4\r\n' 'X-Timestamp: 1364456113.00004\r\n' '\r\n' '1\r\n4' 'DELETE /a/c/o5\r\n' 'X-Timestamp: 1364456113.00005\r\n' '\r\n' 'DELETE /a/c/o6\r\n' 'X-Timestamp: 1364456113.00006\r\n' '\r\n') resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ':UPDATES: START', ':UPDATES: END']) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) req = _requests.pop(0) self.assertEqual(req.method, 'PUT') self.assertEqual(req.path, '/device/partition/a/c/o1') self.assertEqual(req.content_length, 1) self.assertEqual(req.headers, { 'Content-Length': '1', 'X-Timestamp': '1364456113.00001', 'X-Object-Meta-Test1': 'one', 'Content-Encoding': 'gzip', 'Specialty-Header': 'value', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp x-object-meta-test1 ' 'content-encoding specialty-header')}) self.assertEqual(req.read_body, '1') req = _requests.pop(0) self.assertEqual(req.method, 'DELETE') self.assertEqual(req.path, '/device/partition/a/c/o2') self.assertEqual(req.headers, { 'X-Timestamp': '1364456113.00002', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': 'x-timestamp'}) req = _requests.pop(0) self.assertEqual(req.method, 'PUT') self.assertEqual(req.path, '/device/partition/a/c/o3') self.assertEqual(req.content_length, 3) self.assertEqual(req.headers, { 'Content-Length': '3', 'X-Timestamp': '1364456113.00003', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp')}) self.assertEqual(req.read_body, '123') req = _requests.pop(0) self.assertEqual(req.method, 'PUT') self.assertEqual(req.path, '/device/partition/a/c/o4') self.assertEqual(req.content_length, 4) self.assertEqual(req.headers, { 'Content-Length': '4', 'X-Timestamp': '1364456113.00004', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp')}) self.assertEqual(req.read_body, '1\r\n4') req = _requests.pop(0) self.assertEqual(req.method, 'DELETE') self.assertEqual(req.path, '/device/partition/a/c/o5') self.assertEqual(req.headers, { 'X-Timestamp': '1364456113.00005', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': 'x-timestamp'}) req = _requests.pop(0) self.assertEqual(req.method, 'DELETE') self.assertEqual(req.path, '/device/partition/a/c/o6') self.assertEqual(req.headers, { 'X-Timestamp': '1364456113.00006', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': 'x-timestamp'}) self.assertEqual(_requests, []) def test_UPDATES_subreq_does_not_read_all(self): # This tests that if a REPLICATION subrequest fails and doesn't read # all the subrequest body that it will read and throw away the rest of # the body before moving on to the next subrequest. # If you comment out the part in ssync_receiver where it does: # for junk in subreq.environ['wsgi.input']: # pass # You can then see this test fail. _requests = [] @server.public def _PUT(request): _requests.append(request) # Deliberately just reading up to first 2 bytes. request.read_body = request.environ['wsgi.input'].read(2) return swob.HTTPInternalServerError() class _IgnoreReadlineHint(StringIO.StringIO): def __init__(self, value): StringIO.StringIO.__init__(self, value) def readline(self, hint=-1): return StringIO.StringIO.readline(self) self.controller.PUT = _PUT self.controller.network_chunk_size = 2 self.controller.logger = mock.MagicMock() req = swob.Request.blank( '/device/partition', environ={'REQUEST_METHOD': 'REPLICATION'}, body=':MISSING_CHECK: START\r\n:MISSING_CHECK: END\r\n' ':UPDATES: START\r\n' 'PUT /a/c/o1\r\n' 'Content-Length: 3\r\n' 'X-Timestamp: 1364456113.00001\r\n' '\r\n' '123' 'PUT /a/c/o2\r\n' 'Content-Length: 1\r\n' 'X-Timestamp: 1364456113.00002\r\n' '\r\n' '1') req.environ['wsgi.input'] = _IgnoreReadlineHint(req.body) resp = req.get_response(self.controller) self.assertEqual( self.body_lines(resp.body), [':MISSING_CHECK: START', ':MISSING_CHECK: END', ":ERROR: 500 'ERROR: With :UPDATES: 2 failures to 0 successes'"]) self.assertEqual(resp.status_int, 200) self.assertFalse(self.controller.logger.exception.called) self.assertFalse(self.controller.logger.error.called) req = _requests.pop(0) self.assertEqual(req.path, '/device/partition/a/c/o1') self.assertEqual(req.content_length, 3) self.assertEqual(req.headers, { 'Content-Length': '3', 'X-Timestamp': '1364456113.00001', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp')}) self.assertEqual(req.read_body, '12') req = _requests.pop(0) self.assertEqual(req.path, '/device/partition/a/c/o2') self.assertEqual(req.content_length, 1) self.assertEqual(req.headers, { 'Content-Length': '1', 'X-Timestamp': '1364456113.00002', 'Host': 'localhost:80', 'X-Backend-Replication': 'True', 'X-Backend-Replication-Headers': ( 'content-length x-timestamp')}) self.assertEqual(req.read_body, '1') self.assertEqual(_requests, []) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_updater.py0000664000175400017540000002600112323703611021651 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import cPickle as pickle import mock import os import unittest from contextlib import closing from gzip import GzipFile from tempfile import mkdtemp from shutil import rmtree from time import time from distutils.dir_util import mkpath from eventlet import spawn, Timeout, listen from swift.obj import updater as object_updater from swift.obj.diskfile import ASYNCDIR from swift.common.ring import RingData from swift.common import utils from swift.common.utils import hash_path, normalize_timestamp, mkdirs, \ write_pickle from test.unit import FakeLogger class TestObjectUpdater(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' self.testdir = mkdtemp() ring_file = os.path.join(self.testdir, 'container.ring.gz') with closing(GzipFile(ring_file, 'wb')) as f: pickle.dump( RingData([[0, 1, 2, 0, 1, 2], [1, 2, 0, 1, 2, 0], [2, 3, 1, 2, 3, 1]], [{'id': 0, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 0}, {'id': 1, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 2}, {'id': 2, 'ip': '127.0.0.1', 'port': 1, 'device': 'sda1', 'zone': 4}], 30), f) self.devices_dir = os.path.join(self.testdir, 'devices') os.mkdir(self.devices_dir) self.sda1 = os.path.join(self.devices_dir, 'sda1') os.mkdir(self.sda1) os.mkdir(os.path.join(self.sda1, 'tmp')) def tearDown(self): rmtree(self.testdir, ignore_errors=1) def test_creation(self): cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '2', 'node_timeout': '5'}) self.assert_(hasattr(cu, 'logger')) self.assert_(cu.logger is not None) self.assertEquals(cu.devices, self.devices_dir) self.assertEquals(cu.interval, 1) self.assertEquals(cu.concurrency, 2) self.assertEquals(cu.node_timeout, 5) self.assert_(cu.get_container_ring() is not None) def test_object_sweep(self): prefix_dir = os.path.join(self.sda1, ASYNCDIR, 'abc') mkpath(prefix_dir) # A non-directory where directory is expected should just be skipped... not_a_dir_path = os.path.join(self.sda1, ASYNCDIR, 'not_a_dir') with open(not_a_dir_path, 'w'): pass objects = { 'a': [1089.3, 18.37, 12.83, 1.3], 'b': [49.4, 49.3, 49.2, 49.1], 'c': [109984.123], } expected = set() for o, timestamps in objects.iteritems(): ohash = hash_path('account', 'container', o) for t in timestamps: o_path = os.path.join(prefix_dir, ohash + '-' + normalize_timestamp(t)) if t == timestamps[0]: expected.add(o_path) write_pickle({}, o_path) seen = set() class MockObjectUpdater(object_updater.ObjectUpdater): def process_object_update(self, update_path, device): seen.add(update_path) os.unlink(update_path) cu = MockObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '5'}) cu.object_sweep(self.sda1) self.assert_(not os.path.exists(prefix_dir)) self.assert_(os.path.exists(not_a_dir_path)) self.assertEqual(expected, seen) @mock.patch.object(object_updater, 'ismount') def test_run_once_with_disk_unmounted(self, mock_ismount): mock_ismount.return_value = False cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) cu.run_once() async_dir = os.path.join(self.sda1, ASYNCDIR) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) # mount_check == False means no call to ismount self.assertEqual([], mock_ismount.mock_calls) cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) odd_dir = os.path.join(async_dir, 'not really supposed to be here') os.mkdir(odd_dir) cu.logger = FakeLogger() cu.run_once() self.assert_(os.path.exists(async_dir)) self.assert_(os.path.exists(odd_dir)) # skipped because not mounted! # mount_check == True means ismount was checked self.assertEqual([ mock.call(self.sda1), ], mock_ismount.mock_calls) self.assertEqual(cu.logger.get_increment_counts(), {'errors': 1}) @mock.patch.object(object_updater, 'ismount') def test_run_once(self, mock_ismount): mock_ismount.return_value = True cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'false', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) cu.run_once() async_dir = os.path.join(self.sda1, ASYNCDIR) os.mkdir(async_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) # mount_check == False means no call to ismount self.assertEqual([], mock_ismount.mock_calls) cu = object_updater.ObjectUpdater({ 'devices': self.devices_dir, 'mount_check': 'TrUe', 'swift_dir': self.testdir, 'interval': '1', 'concurrency': '1', 'node_timeout': '15'}) odd_dir = os.path.join(async_dir, 'not really supposed to be here') os.mkdir(odd_dir) cu.run_once() self.assert_(os.path.exists(async_dir)) self.assert_(not os.path.exists(odd_dir)) # mount_check == True means ismount was checked self.assertEqual([ mock.call(self.sda1), ], mock_ismount.mock_calls) ohash = hash_path('a', 'c', 'o') odir = os.path.join(async_dir, ohash[-3:]) mkdirs(odir) older_op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time() - 1))) op_path = os.path.join( odir, '%s-%s' % (ohash, normalize_timestamp(time()))) for path in (op_path, older_op_path): with open(path, 'wb') as async_pending: pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', 'headers': { 'X-Container-Timestamp': normalize_timestamp(0)}}, async_pending) cu.logger = FakeLogger() cu.run_once() self.assert_(not os.path.exists(older_op_path)) self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1, 'unlinks': 1}) self.assertEqual(None, pickle.load(open(op_path)).get('successes')) bindsock = listen(('127.0.0.1', 0)) def accepter(sock, return_code): try: with Timeout(3): inc = sock.makefile('rb') out = sock.makefile('wb') out.write('HTTP/1.1 %d OK\r\nContent-Length: 0\r\n\r\n' % return_code) out.flush() self.assertEquals(inc.readline(), 'PUT /sda1/0/a/c/o HTTP/1.1\r\n') headers = {} line = inc.readline() while line and line != '\r\n': headers[line.split(':')[0].lower()] = \ line.split(':')[1].strip() line = inc.readline() self.assert_('x-container-timestamp' in headers) except BaseException as err: return err return None def accept(return_codes): codes = iter(return_codes) try: events = [] for x in xrange(len(return_codes)): with Timeout(3): sock, addr = bindsock.accept() events.append( spawn(accepter, sock, codes.next())) for event in events: err = event.wait() if err: raise err except BaseException as err: return err return None event = spawn(accept, [201, 500, 500]) for dev in cu.get_container_ring().devs: if dev is not None: dev['port'] = bindsock.getsockname()[1] cu.logger = FakeLogger() cu.run_once() err = event.wait() if err: raise err self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [404, 500]) cu.logger = FakeLogger() cu.run_once() err = event.wait() if err: raise err self.assert_(os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'failures': 1}) self.assertEqual([0, 1], pickle.load(open(op_path)).get('successes')) event = spawn(accept, [201]) cu.logger = FakeLogger() cu.run_once() err = event.wait() if err: raise err self.assert_(not os.path.exists(op_path)) self.assertEqual(cu.logger.get_increment_counts(), {'unlinks': 1, 'successes': 1}) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_ssync_sender.py0000664000175400017540000007754712323703611022731 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import os import shutil import StringIO import tempfile import time import unittest import eventlet import mock from swift.common import exceptions, utils from swift.obj import ssync_sender, diskfile from test.unit import DebugLogger class FakeReplicator(object): def __init__(self, testdir): self.logger = mock.MagicMock() self.conn_timeout = 1 self.node_timeout = 2 self.http_timeout = 3 self.network_chunk_size = 65536 self.disk_chunk_size = 4096 conf = { 'devices': testdir, 'mount_check': 'false', } self._diskfile_mgr = diskfile.DiskFileManager(conf, DebugLogger()) class NullBufferedHTTPConnection(object): def __init__(*args, **kwargs): pass def putrequest(*args, **kwargs): pass def putheader(*args, **kwargs): pass def endheaders(*args, **kwargs): pass def getresponse(*args, **kwargs): pass class FakeResponse(object): def __init__(self, chunk_body=''): self.status = 200 self.close_called = False if chunk_body: self.fp = StringIO.StringIO( '%x\r\n%s\r\n0\r\n\r\n' % (len(chunk_body), chunk_body)) def close(self): self.close_called = True class FakeConnection(object): def __init__(self): self.sent = [] self.closed = False def send(self, data): self.sent.append(data) def close(self): self.closed = True class TestSender(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() self.testdir = os.path.join(self.tmpdir, 'tmp_test_ssync_sender') self.replicator = FakeReplicator(self.testdir) self.sender = ssync_sender.Sender(self.replicator, None, None, None) def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=1) def _make_open_diskfile(self, device='dev', partition='9', account='a', container='c', obj='o', body='test', extra_metadata=None): object_parts = account, container, obj req_timestamp = utils.normalize_timestamp(time.time()) df = self.sender.daemon._diskfile_mgr.get_diskfile(device, partition, *object_parts) content_length = len(body) etag = hashlib.md5(body).hexdigest() with df.create() as writer: writer.write(body) metadata = { 'X-Timestamp': req_timestamp, 'Content-Length': content_length, 'ETag': etag, } if extra_metadata: metadata.update(extra_metadata) writer.put(metadata) df.open() return df def test_call_catches_MessageTimeout(self): def connect(self): exc = exceptions.MessageTimeout(1, 'test connect') # Cancels Eventlet's raising of this since we're about to do it. exc.cancel() raise exc with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual( call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '1 second: test connect') def test_call_catches_ReplicationException(self): def connect(self): raise exceptions.ReplicationException('test connect') with mock.patch.object(ssync_sender.Sender, 'connect', connect): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual( call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), 'test connect') def test_call_catches_other_exceptions(self): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' self.assertFalse(self.sender()) call = self.replicator.logger.exception.mock_calls[0] self.assertEqual( call[1], ('%s:%s/%s/%s EXCEPTION in replication.Sender', '1.2.3.4', 5678, 'sda1', '9')) def test_call_catches_exception_handling_exception(self): node = dict(ip='1.2.3.4', port=5678, device='sda1') job = None # Will cause inside exception handler to fail self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] self.sender.connect = 'cause exception' self.assertFalse(self.sender()) self.replicator.logger.exception.assert_called_once_with( 'EXCEPTION in replication.Sender') def test_call_calls_others(self): self.sender.suffixes = ['abc'] self.sender.connect = mock.MagicMock() self.sender.missing_check = mock.MagicMock() self.sender.updates = mock.MagicMock() self.sender.disconnect = mock.MagicMock() self.assertTrue(self.sender()) self.sender.connect.assert_called_once_with() self.sender.missing_check.assert_called_once_with() self.sender.updates.assert_called_once_with() self.sender.disconnect.assert_called_once_with() def test_call_calls_others_returns_failure(self): self.sender.suffixes = ['abc'] self.sender.connect = mock.MagicMock() self.sender.missing_check = mock.MagicMock() self.sender.updates = mock.MagicMock() self.sender.disconnect = mock.MagicMock() self.sender.failures = 1 self.assertFalse(self.sender()) self.sender.connect.assert_called_once_with() self.sender.missing_check.assert_called_once_with() self.sender.updates.assert_called_once_with() self.sender.disconnect.assert_called_once_with() def test_connect_send_timeout(self): self.replicator.conn_timeout = 0.01 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] def putrequest(*args, **kwargs): eventlet.sleep(0.1) with mock.patch.object( ssync_sender.bufferedhttp.BufferedHTTPConnection, 'putrequest', putrequest): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual( call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '0.01 seconds: connect send') def test_connect_receive_timeout(self): self.replicator.node_timeout = 0.02 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): def getresponse(*args, **kwargs): eventlet.sleep(0.1) with mock.patch.object( ssync_sender.bufferedhttp, 'BufferedHTTPConnection', FakeBufferedHTTPConnection): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual( call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), '0.02 seconds: connect receive') def test_connect_bad_status(self): self.replicator.node_timeout = 0.02 node = dict(ip='1.2.3.4', port=5678, device='sda1') job = dict(partition='9') self.sender = ssync_sender.Sender(self.replicator, node, job, None) self.sender.suffixes = ['abc'] class FakeBufferedHTTPConnection(NullBufferedHTTPConnection): def getresponse(*args, **kwargs): response = FakeResponse() response.status = 503 return response with mock.patch.object( ssync_sender.bufferedhttp, 'BufferedHTTPConnection', FakeBufferedHTTPConnection): self.assertFalse(self.sender()) call = self.replicator.logger.error.mock_calls[0] self.assertEqual( call[1][:-1], ('%s:%s/%s/%s %s', '1.2.3.4', 5678, 'sda1', '9')) self.assertEqual(str(call[1][-1]), 'Expected status 200; got 503') def test_readline_newline_in_buffer(self): self.sender.response_buffer = 'Has a newline already.\r\nOkay.' self.assertEqual(self.sender.readline(), 'Has a newline already.\r\n') self.assertEqual(self.sender.response_buffer, 'Okay.') def test_readline_buffer_exceeds_network_chunk_size_somehow(self): self.replicator.network_chunk_size = 2 self.sender.response_buffer = '1234567890' self.assertEqual(self.sender.readline(), '1234567890') self.assertEqual(self.sender.response_buffer, '') def test_readline_at_start_of_chunk(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO('2\r\nx\n\r\n') self.assertEqual(self.sender.readline(), 'x\n') def test_readline_chunk_with_extension(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO( '2 ; chunk=extension\r\nx\n\r\n') self.assertEqual(self.sender.readline(), 'x\n') def test_readline_broken_chunk(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO('q\r\nx\n\r\n') self.assertRaises( exceptions.ReplicationException, self.sender.readline) self.assertTrue(self.sender.response.close_called) def test_readline_terminated_chunk(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO('b\r\nnot enough') self.assertRaises( exceptions.ReplicationException, self.sender.readline) self.assertTrue(self.sender.response.close_called) def test_readline_all(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO('2\r\nx\n\r\n0\r\n\r\n') self.assertEqual(self.sender.readline(), 'x\n') self.assertEqual(self.sender.readline(), '') self.assertEqual(self.sender.readline(), '') def test_readline_all_trailing_not_newline_termed(self): self.sender.response = FakeResponse() self.sender.response.fp = StringIO.StringIO( '2\r\nx\n\r\n3\r\n123\r\n0\r\n\r\n') self.assertEqual(self.sender.readline(), 'x\n') self.assertEqual(self.sender.readline(), '123') self.assertEqual(self.sender.readline(), '') self.assertEqual(self.sender.readline(), '') def test_missing_check_timeout(self): self.sender.connection = FakeConnection() self.sender.connection.send = lambda d: eventlet.sleep(1) self.sender.daemon.node_timeout = 0.01 self.assertRaises(exceptions.MessageTimeout, self.sender.missing_check) def test_missing_check_has_empty_suffixes(self): def yield_hashes(device, partition, suffixes=None): if device != 'dev' or partition != '9' or suffixes != [ 'abc', 'def']: yield # Just here to make this a generator raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc', 'def'] self.sender.response = FakeResponse( chunk_body=( ':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n')) self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.missing_check() self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, []) def test_missing_check_has_suffixes(self): def yield_hashes(device, partition, suffixes=None): if device == 'dev' and partition == '9' and suffixes == [ 'abc', 'def']: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') yield ( '/srv/node/dev/objects/9/def/' '9d41d8cd98f00b204e9800998ecf0def', '9d41d8cd98f00b204e9800998ecf0def', '1380144472.22222') yield ( '/srv/node/dev/objects/9/def/' '9d41d8cd98f00b204e9800998ecf1def', '9d41d8cd98f00b204e9800998ecf1def', '1380144474.44444') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc', 'def'] self.sender.response = FakeResponse( chunk_body=( ':MISSING_CHECK: START\r\n' ':MISSING_CHECK: END\r\n')) self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.missing_check() self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0def 1380144472.22222\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf1def 1380144474.44444\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, []) def test_missing_check_far_end_disconnect(self): def yield_hashes(device, partition, suffixes=None): if device == 'dev' and partition == '9' and suffixes == ['abc']: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse(chunk_body='\r\n') exc = None try: self.sender.missing_check() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), 'Early disconnect') self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') def test_missing_check_far_end_disconnect2(self): def yield_hashes(device, partition, suffixes=None): if device == 'dev' and partition == '9' and suffixes == ['abc']: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse( chunk_body=':MISSING_CHECK: START\r\n') exc = None try: self.sender.missing_check() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), 'Early disconnect') self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') def test_missing_check_far_end_unexpected(self): def yield_hashes(device, partition, suffixes=None): if device == 'dev' and partition == '9' and suffixes == ['abc']: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc'] self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.response = FakeResponse(chunk_body='OH HAI\r\n') exc = None try: self.sender.missing_check() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), "Unexpected response: 'OH HAI'") self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') def test_missing_check_send_list(self): def yield_hashes(device, partition, suffixes=None): if device == 'dev' and partition == '9' and suffixes == ['abc']: yield ( '/srv/node/dev/objects/9/abc/' '9d41d8cd98f00b204e9800998ecf0abc', '9d41d8cd98f00b204e9800998ecf0abc', '1380144470.00000') else: raise Exception( 'No match for %r %r %r' % (device, partition, suffixes)) self.sender.connection = FakeConnection() self.sender.job = {'device': 'dev', 'partition': '9'} self.sender.suffixes = ['abc'] self.sender.response = FakeResponse( chunk_body=( ':MISSING_CHECK: START\r\n' '0123abc\r\n' ':MISSING_CHECK: END\r\n')) self.sender.daemon._diskfile_mgr.yield_hashes = yield_hashes self.sender.missing_check() self.assertEqual( ''.join(self.sender.connection.sent), '17\r\n:MISSING_CHECK: START\r\n\r\n' '33\r\n9d41d8cd98f00b204e9800998ecf0abc 1380144470.00000\r\n\r\n' '15\r\n:MISSING_CHECK: END\r\n\r\n') self.assertEqual(self.sender.send_list, ['0123abc']) def test_updates_timeout(self): self.sender.connection = FakeConnection() self.sender.connection.send = lambda d: eventlet.sleep(1) self.sender.daemon.node_timeout = 0.01 self.assertRaises(exceptions.MessageTimeout, self.sender.updates) def test_updates_empty_send_list(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_unexpected_response_lines1(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( 'abc\r\n' ':UPDATES: START\r\n' ':UPDATES: END\r\n')) exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), "Unexpected response: 'abc'") self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_unexpected_response_lines2(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' 'abc\r\n' ':UPDATES: END\r\n')) exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), "Unexpected response: 'abc'") self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_is_deleted(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts) object_hash = utils.hash_path(*object_parts) delete_timestamp = utils.normalize_timestamp(time.time()) df.delete(delete_timestamp) self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() self.sender.send_put = mock.MagicMock() self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.sender.send_delete.assert_called_once_with( '/a/c/o', delete_timestamp) self.assertEqual(self.sender.send_put.mock_calls, []) # note that the delete line isn't actually sent since we mock # send_delete; send_delete is tested separately. self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_put(self): device = 'dev' part = '9' object_parts = ('a', 'c', 'o') df = self._make_open_diskfile(device, part, *object_parts) object_hash = utils.hash_path(*object_parts) expected = df.get_metadata() self.sender.connection = FakeConnection() self.sender.job = {'device': device, 'partition': part} self.sender.node = {} self.sender.send_list = [object_hash] self.sender.send_delete = mock.MagicMock() self.sender.send_put = mock.MagicMock() self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) self.sender.updates() self.assertEqual(self.sender.send_delete.mock_calls, []) self.assertEqual(1, len(self.sender.send_put.mock_calls)) args, _kwargs = self.sender.send_put.call_args path, df = args self.assertEqual(path, '/a/c/o') self.assert_(isinstance(df, diskfile.DiskFile)) self.assertEqual(expected, df.get_metadata()) # note that the put line isn't actually sent since we mock send_put; # send_put is tested separately. self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_read_response_timeout_start(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) orig_readline = self.sender.readline def delayed_readline(): eventlet.sleep(1) return orig_readline() self.sender.readline = delayed_readline self.sender.daemon.http_timeout = 0.01 self.assertRaises(exceptions.MessageTimeout, self.sender.updates) def test_updates_read_response_disconnect_start(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse(chunk_body='\r\n') exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), 'Early disconnect') self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_read_response_unexp_start(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( 'anything else\r\n' ':UPDATES: START\r\n' ':UPDATES: END\r\n')) exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), "Unexpected response: 'anything else'") self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_read_response_timeout_end(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' ':UPDATES: END\r\n')) orig_readline = self.sender.readline def delayed_readline(): rv = orig_readline() if rv == ':UPDATES: END\r\n': eventlet.sleep(1) return rv self.sender.readline = delayed_readline self.sender.daemon.http_timeout = 0.01 self.assertRaises(exceptions.MessageTimeout, self.sender.updates) def test_updates_read_response_disconnect_end(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' '\r\n')) exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), 'Early disconnect') self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_updates_read_response_unexp_end(self): self.sender.connection = FakeConnection() self.sender.send_list = [] self.sender.response = FakeResponse( chunk_body=( ':UPDATES: START\r\n' 'anything else\r\n' ':UPDATES: END\r\n')) exc = None try: self.sender.updates() except exceptions.ReplicationException as err: exc = err self.assertEqual(str(exc), "Unexpected response: 'anything else'") self.assertEqual( ''.join(self.sender.connection.sent), '11\r\n:UPDATES: START\r\n\r\n' 'f\r\n:UPDATES: END\r\n\r\n') def test_send_delete_timeout(self): self.sender.connection = FakeConnection() self.sender.connection.send = lambda d: eventlet.sleep(1) self.sender.daemon.node_timeout = 0.01 exc = None try: self.sender.send_delete('/a/c/o', '1381679759.90941') except exceptions.MessageTimeout as err: exc = err self.assertEqual(str(exc), '0.01 seconds: send_delete') def test_send_delete(self): self.sender.connection = FakeConnection() self.sender.send_delete('/a/c/o', '1381679759.90941') self.assertEqual( ''.join(self.sender.connection.sent), '30\r\n' 'DELETE /a/c/o\r\n' 'X-Timestamp: 1381679759.90941\r\n' '\r\n\r\n') def test_send_put_initial_timeout(self): df = self._make_open_diskfile() df._disk_chunk_size = 2 self.sender.connection = FakeConnection() self.sender.connection.send = lambda d: eventlet.sleep(1) self.sender.daemon.node_timeout = 0.01 exc = None try: self.sender.send_put('/a/c/o', df) except exceptions.MessageTimeout as err: exc = err self.assertEqual(str(exc), '0.01 seconds: send_put') def test_send_put_chunk_timeout(self): df = self._make_open_diskfile() self.sender.connection = FakeConnection() self.sender.daemon.node_timeout = 0.01 one_shot = [None] def mock_send(data): try: one_shot.pop() except IndexError: eventlet.sleep(1) self.sender.connection.send = mock_send exc = None try: self.sender.send_put('/a/c/o', df) except exceptions.MessageTimeout as err: exc = err self.assertEqual(str(exc), '0.01 seconds: send_put chunk') def test_send_put(self): body = 'test' extra_metadata = {'Some-Other-Header': 'value'} df = self._make_open_diskfile(body=body, extra_metadata=extra_metadata) expected = dict(df.get_metadata()) expected['body'] = body expected['chunk_size'] = len(body) self.sender.connection = FakeConnection() self.sender.send_put('/a/c/o', df) self.assertEqual( ''.join(self.sender.connection.sent), '82\r\n' 'PUT /a/c/o\r\n' 'Content-Length: %(Content-Length)s\r\n' 'ETag: %(ETag)s\r\n' 'Some-Other-Header: value\r\n' 'X-Timestamp: %(X-Timestamp)s\r\n' '\r\n' '\r\n' '%(chunk_size)s\r\n' '%(body)s\r\n' % expected) def test_disconnect_timeout(self): self.sender.connection = FakeConnection() self.sender.connection.send = lambda d: eventlet.sleep(1) self.sender.daemon.node_timeout = 0.01 self.sender.disconnect() self.assertEqual(''.join(self.sender.connection.sent), '') self.assertTrue(self.sender.connection.closed) def test_disconnect(self): self.sender.connection = FakeConnection() self.sender.disconnect() self.assertEqual(''.join(self.sender.connection.sent), '0\r\n\r\n') self.assertTrue(self.sender.connection.closed) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/test_diskfile.py0000664000175400017540000024615712323703611022017 0ustar jenkinsjenkins00000000000000#-*- coding:utf-8 -*- # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for swift.obj.diskfile""" import cPickle as pickle import os import errno import mock import unittest import email import tempfile import uuid import xattr from shutil import rmtree from time import time from tempfile import mkdtemp from hashlib import md5 from contextlib import closing, nested from gzip import GzipFile from eventlet import tpool from test.unit import FakeLogger, mock as unit_mock, temptree from swift.obj import diskfile from swift.common import utils from swift.common.utils import hash_path, mkdirs, normalize_timestamp from swift.common import ring from swift.common.exceptions import DiskFileNotExist, DiskFileQuarantined, \ DiskFileDeviceUnavailable, DiskFileDeleted, DiskFileNotOpen, \ DiskFileError, ReplicationLockTimeout, PathNotDir, DiskFileCollision, \ DiskFileExpired, SwiftException, DiskFileNoSpace def _create_test_ring(path): testgz = os.path.join(path, 'object.ring.gz') intended_replica2part2dev_id = [ [0, 1, 2, 3, 4, 5, 6], [1, 2, 3, 0, 5, 6, 4], [2, 3, 0, 1, 6, 4, 5]] intended_devs = [ {'id': 0, 'device': 'sda', 'zone': 0, 'ip': '127.0.0.0', 'port': 6000}, {'id': 1, 'device': 'sda', 'zone': 1, 'ip': '127.0.0.1', 'port': 6000}, {'id': 2, 'device': 'sda', 'zone': 2, 'ip': '127.0.0.2', 'port': 6000}, {'id': 3, 'device': 'sda', 'zone': 4, 'ip': '127.0.0.3', 'port': 6000}, {'id': 4, 'device': 'sda', 'zone': 5, 'ip': '127.0.0.4', 'port': 6000}, {'id': 5, 'device': 'sda', 'zone': 6, 'ip': 'fe80::202:b3ff:fe1e:8329', 'port': 6000}, {'id': 6, 'device': 'sda', 'zone': 7, 'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', 'port': 6000}] intended_part_shift = 30 intended_reload_time = 15 with closing(GzipFile(testgz, 'wb')) as f: pickle.dump( ring.RingData(intended_replica2part2dev_id, intended_devs, intended_part_shift), f) return ring.Ring(path, ring_name='object', reload_time=intended_reload_time) class TestDiskFileModuleMethods(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = '' # Setup a test ring (stolen from common/test_ring.py) self.testdir = tempfile.mkdtemp() self.devices = os.path.join(self.testdir, 'node') rmtree(self.testdir, ignore_errors=1) os.mkdir(self.testdir) os.mkdir(self.devices) self.existing_device = 'sda' os.mkdir(os.path.join(self.devices, self.existing_device)) self.objects = os.path.join(self.devices, self.existing_device, 'objects') os.mkdir(self.objects) self.parts = {} for part in ['0', '1', '2', '3']: self.parts[part] = os.path.join(self.objects, part) os.mkdir(os.path.join(self.objects, part)) self.ring = _create_test_ring(self.testdir) self.conf = dict( swift_dir=self.testdir, devices=self.devices, mount_check='false', timeout='300', stats_interval='1') self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) def tearDown(self): rmtree(self.testdir, ignore_errors=1) def _create_diskfile(self): return self.df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o') def test_quarantine_renamer(self): # we use this for convenience, not really about a diskfile layout df = self._create_diskfile() mkdirs(df._datadir) exp_dir = os.path.join(self.devices, 'quarantined', 'objects', os.path.basename(df._datadir)) qbit = os.path.join(df._datadir, 'qbit') with open(qbit, 'w') as f: f.write('abc') to_dir = diskfile.quarantine_renamer(self.devices, qbit) self.assertEqual(to_dir, exp_dir) self.assertRaises(OSError, diskfile.quarantine_renamer, self.devices, qbit) def test_hash_suffix_enoent(self): self.assertRaises(PathNotDir, diskfile.hash_suffix, os.path.join(self.testdir, "doesnotexist"), 101) def test_hash_suffix_oserror(self): mocked_os_listdir = mock.Mock( side_effect=OSError(errno.EACCES, os.strerror(errno.EACCES))) with mock.patch("os.listdir", mocked_os_listdir): self.assertRaises(OSError, diskfile.hash_suffix, os.path.join(self.testdir, "doesnotexist"), 101) def test_hash_suffix_hash_dir_is_file_quarantine(self): df = self._create_diskfile() mkdirs(os.path.dirname(df._datadir)) open(df._datadir, 'wb').close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) orig_quarantine_renamer = diskfile.quarantine_renamer called = [False] def wrapped(*args, **kwargs): called[0] = True return orig_quarantine_renamer(*args, **kwargs) try: diskfile.quarantine_renamer = wrapped diskfile.hash_suffix(whole_path_from, 101) finally: diskfile.quarantine_renamer = orig_quarantine_renamer self.assertTrue(called[0]) def test_hash_suffix_one_file(self): df = self._create_diskfile() mkdirs(df._datadir) f = open( os.path.join(df._datadir, normalize_timestamp(time() - 100) + '.ts'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) diskfile.hash_suffix(whole_path_from, 101) self.assertEquals(len(os.listdir(self.parts['0'])), 1) diskfile.hash_suffix(whole_path_from, 99) self.assertEquals(len(os.listdir(self.parts['0'])), 0) def test_hash_suffix_oserror_on_hcl(self): df = self._create_diskfile() mkdirs(df._datadir) f = open( os.path.join(df._datadir, normalize_timestamp(time() - 100) + '.ts'), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) state = [0] orig_os_listdir = os.listdir def mock_os_listdir(*args, **kwargs): # We want the first call to os.listdir() to succeed, which is the # one directly from hash_suffix() itself, but then we want to fail # the next call to os.listdir() which is from # hash_cleanup_listdir() if state[0] == 1: raise OSError(errno.EACCES, os.strerror(errno.EACCES)) state[0] = 1 return orig_os_listdir(*args, **kwargs) with mock.patch('os.listdir', mock_os_listdir): self.assertRaises(OSError, diskfile.hash_suffix, whole_path_from, 101) def test_hash_suffix_multi_file_one(self): df = self._create_diskfile() mkdirs(df._datadir) for tdiff in [1, 50, 100, 500]: for suff in ['.meta', '.data', '.ts']: f = open( os.path.join( df._datadir, normalize_timestamp(int(time()) - tdiff) + suff), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hsh_path = os.listdir(whole_path_from)[0] whole_hsh_path = os.path.join(whole_path_from, hsh_path) diskfile.hash_suffix(whole_path_from, 99) # only the tombstone should be left self.assertEquals(len(os.listdir(whole_hsh_path)), 1) def test_hash_suffix_multi_file_two(self): df = self._create_diskfile() mkdirs(df._datadir) for tdiff in [1, 50, 100, 500]: suffs = ['.meta', '.data'] if tdiff > 50: suffs.append('.ts') for suff in suffs: f = open( os.path.join( df._datadir, normalize_timestamp(int(time()) - tdiff) + suff), 'wb') f.write('1234567890') f.close() ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hsh_path = os.listdir(whole_path_from)[0] whole_hsh_path = os.path.join(whole_path_from, hsh_path) diskfile.hash_suffix(whole_path_from, 99) # only the meta and data should be left self.assertEquals(len(os.listdir(whole_hsh_path)), 2) def test_hash_suffix_hsh_path_disappearance(self): orig_rmdir = os.rmdir def _rmdir(path): # Done twice to recreate what happens when it doesn't exist. orig_rmdir(path) orig_rmdir(path) df = self.df_mgr.get_diskfile('sda', '0', 'a', 'c', 'o') mkdirs(df._datadir) ohash = hash_path('a', 'c', 'o') suffix = ohash[-3:] suffix_path = os.path.join(self.objects, '0', suffix) with mock.patch('os.rmdir', _rmdir): # If hash_suffix doesn't handle the exception _rmdir will raise, # this test will fail. diskfile.hash_suffix(suffix_path, 123) def test_invalidate_hash(self): def assertFileData(file_path, data): with open(file_path, 'r') as fp: fdata = fp.read() self.assertEquals(pickle.loads(fdata), pickle.loads(data)) df = self._create_diskfile() mkdirs(df._datadir) ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hashes_file = os.path.join(self.objects, '0', diskfile.HASH_FILE) # test that non existent file except caught self.assertEquals(diskfile.invalidate_hash(whole_path_from), None) # test that hashes get cleared check_pickle_data = pickle.dumps({data_dir: None}, diskfile.PICKLE_PROTOCOL) for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: with open(hashes_file, 'wb') as fp: pickle.dump(data_hash, fp, diskfile.PICKLE_PROTOCOL) diskfile.invalidate_hash(whole_path_from) assertFileData(hashes_file, check_pickle_data) def test_invalidate_hash_bad_pickle(self): df = self._create_diskfile() mkdirs(df._datadir) ohash = hash_path('a', 'c', 'o') data_dir = ohash[-3:] whole_path_from = os.path.join(self.objects, '0', data_dir) hashes_file = os.path.join(self.objects, '0', diskfile.HASH_FILE) for data_hash in [{data_dir: None}, {data_dir: 'abcdefg'}]: with open(hashes_file, 'wb') as fp: fp.write('bad hash data') try: diskfile.invalidate_hash(whole_path_from) except Exception as err: self.fail("Unexpected exception raised: %s" % err) else: pass def test_get_hashes(self): df = self._create_diskfile() mkdirs(df._datadir) with open( os.path.join(df._datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes = diskfile.get_hashes(part) self.assertEquals(hashed, 1) self.assert_('a83' in hashes) hashed, hashes = diskfile.get_hashes(part, do_listdir=True) self.assertEquals(hashed, 0) self.assert_('a83' in hashes) hashed, hashes = diskfile.get_hashes(part, recalculate=['a83']) self.assertEquals(hashed, 1) self.assert_('a83' in hashes) def test_get_hashes_bad_dir(self): df = self._create_diskfile() mkdirs(df._datadir) with open(os.path.join(self.objects, '0', 'bad'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes = diskfile.get_hashes(part) self.assertEquals(hashed, 1) self.assert_('a83' in hashes) self.assert_('bad' not in hashes) def test_get_hashes_unmodified(self): df = self._create_diskfile() mkdirs(df._datadir) with open( os.path.join(df._datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes = diskfile.get_hashes(part) i = [0] def _getmtime(filename): i[0] += 1 return 1 with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): hashed, hashes = diskfile.get_hashes( part, recalculate=['a83']) self.assertEquals(i[0], 2) def test_get_hashes_unmodified_norecalc(self): df = self._create_diskfile() mkdirs(df._datadir) with open( os.path.join(df._datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes_0 = diskfile.get_hashes(part) self.assertEqual(hashed, 1) self.assertTrue('a83' in hashes_0) hashed, hashes_1 = diskfile.get_hashes(part) self.assertEqual(hashed, 0) self.assertTrue('a83' in hashes_0) self.assertEqual(hashes_1, hashes_0) def test_get_hashes_hash_suffix_error(self): df = self._create_diskfile() mkdirs(df._datadir) with open( os.path.join(df._datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') mocked_hash_suffix = mock.MagicMock( side_effect=OSError(errno.EACCES, os.strerror(errno.EACCES))) with mock.patch('swift.obj.diskfile.hash_suffix', mocked_hash_suffix): hashed, hashes = diskfile.get_hashes(part) self.assertEqual(hashed, 0) self.assertEqual(hashes, {'a83': None}) def test_get_hashes_unmodified_and_zero_bytes(self): df = self._create_diskfile() mkdirs(df._datadir) part = os.path.join(self.objects, '0') open(os.path.join(part, diskfile.HASH_FILE), 'w') # Now the hash file is zero bytes. i = [0] def _getmtime(filename): i[0] += 1 return 1 with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): hashed, hashes = diskfile.get_hashes( part, recalculate=[]) # getmtime will actually not get called. Initially, the pickle.load # will raise an exception first and later, force_rewrite will # short-circuit the if clause to determine whether to write out a # fresh hashes_file. self.assertEquals(i[0], 0) self.assertTrue('a83' in hashes) def test_get_hashes_modified(self): df = self._create_diskfile() mkdirs(df._datadir) with open( os.path.join(df._datadir, normalize_timestamp(time()) + '.ts'), 'wb') as f: f.write('1234567890') part = os.path.join(self.objects, '0') hashed, hashes = diskfile.get_hashes(part) i = [0] def _getmtime(filename): if i[0] < 3: i[0] += 1 return i[0] with unit_mock({'swift.obj.diskfile.getmtime': _getmtime}): hashed, hashes = diskfile.get_hashes( part, recalculate=['a83']) self.assertEquals(i[0], 3) def check_hash_cleanup_listdir(self, input_files, output_files): orig_unlink = os.unlink file_list = list(input_files) def mock_listdir(path): return list(file_list) def mock_unlink(path): # timestamp 1 is a special tag to pretend a file disappeared while # working. if '/0000000001.00000.' in path: # Using actual os.unlink to reproduce exactly what OSError it # raises. orig_unlink(uuid.uuid4().hex) file_list.remove(os.path.basename(path)) with unit_mock({'os.listdir': mock_listdir, 'os.unlink': mock_unlink}): self.assertEquals(diskfile.hash_cleanup_listdir('/whatever'), output_files) def test_hash_cleanup_listdir_purge_data_newer_ts(self): # purge .data if there's a newer .ts file1 = normalize_timestamp(time()) + '.data' file2 = normalize_timestamp(time() + 1) + '.ts' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_purge_ts_newer_data(self): # purge .ts if there's a newer .data file1 = normalize_timestamp(time()) + '.ts' file2 = normalize_timestamp(time() + 1) + '.data' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) def test_hash_cleanup_listdir_keep_meta_data_purge_ts(self): # keep .meta and .data if meta newer than data and purge .ts file1 = normalize_timestamp(time()) + '.ts' file2 = normalize_timestamp(time() + 1) + '.data' file3 = normalize_timestamp(time() + 2) + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_keep_one_ts(self): # keep only latest of multiple .ts files file1 = normalize_timestamp(time()) + '.ts' file2 = normalize_timestamp(time() + 1) + '.ts' file3 = normalize_timestamp(time() + 2) + '.ts' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_data(self): # keep only latest of multiple .data files file1 = normalize_timestamp(time()) + '.data' file2 = normalize_timestamp(time() + 1) + '.data' file3 = normalize_timestamp(time() + 2) + '.data' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3]) def test_hash_cleanup_listdir_keep_one_meta(self): # keep only latest of multiple .meta files file1 = normalize_timestamp(time()) + '.data' file2 = normalize_timestamp(time() + 1) + '.meta' file3 = normalize_timestamp(time() + 2) + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file1]) def test_hash_cleanup_listdir_ignore_orphaned_ts(self): # A more recent orphaned .meta file will prevent old .ts files # from being cleaned up otherwise file1 = normalize_timestamp(time()) + '.ts' file2 = normalize_timestamp(time() + 1) + '.ts' file3 = normalize_timestamp(time() + 2) + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_data_only(self): # Oldest .data will be purge, .meta and .ts won't be touched file1 = normalize_timestamp(time()) + '.data' file2 = normalize_timestamp(time() + 1) + '.ts' file3 = normalize_timestamp(time() + 2) + '.meta' file_list = [file1, file2, file3] self.check_hash_cleanup_listdir(file_list, [file3, file2]) def test_hash_cleanup_listdir_purge_old_ts(self): # A single old .ts file will be removed file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' file_list = [file1] self.check_hash_cleanup_listdir(file_list, []) def test_hash_cleanup_listdir_meta_keeps_old_ts(self): # An orphaned .meta will not clean up a very old .ts file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.ts' file2 = normalize_timestamp(time() + 2) + '.meta' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2, file1]) def test_hash_cleanup_listdir_keep_single_old_data(self): # A single old .data file will not be removed file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.data' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) def test_hash_cleanup_listdir_keep_single_old_meta(self): # A single old .meta file will not be removed file1 = normalize_timestamp(time() - (diskfile.ONE_WEEK + 1)) + '.meta' file_list = [file1] self.check_hash_cleanup_listdir(file_list, [file1]) def test_hash_cleanup_listdir_disappeared_path(self): # Next line listing a non-existent dir used to propagate the OSError; # now should mute that. self.assertEqual(diskfile.hash_cleanup_listdir(uuid.uuid4().hex), []) def test_hash_cleanup_listdir_disappeared_before_unlink_1(self): # Timestamp 1 makes other test routines pretend the file disappeared # while working. file1 = '0000000001.00000.ts' file_list = [file1] self.check_hash_cleanup_listdir(file_list, []) def test_hash_cleanup_listdir_disappeared_before_unlink_2(self): # Timestamp 1 makes other test routines pretend the file disappeared # while working. file1 = '0000000001.00000.data' file2 = '0000000002.00000.ts' file_list = [file1, file2] self.check_hash_cleanup_listdir(file_list, [file2]) class TestObjectAuditLocationGenerator(unittest.TestCase): def _make_file(self, path): try: os.makedirs(os.path.dirname(path)) except OSError as err: if err.errno != errno.EEXIST: raise with open(path, 'w'): pass def test_audit_location_class(self): al = diskfile.AuditLocation('abc', '123', '_-_') self.assertEqual(str(al), 'abc') def test_finding_of_hashdirs(self): with temptree([]) as tmpdir: # the good os.makedirs(os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "5c1fdc1ffb12e5eaf84edc30d8b67aca")) os.makedirs(os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "fdfd184d39080020bc8b487f8a7beaca")) os.makedirs(os.path.join(tmpdir, "sdp", "objects", "1519", "df2", "b0fe7af831cc7b1af5bf486b1c841df2")) os.makedirs(os.path.join(tmpdir, "sdp", "objects", "9720", "ca5", "4a943bc72c2e647c4675923d58cf4ca5")) os.makedirs(os.path.join(tmpdir, "sdq", "objects", "3071", "8eb", "fcd938702024c25fef6c32fef05298eb")) # the bad self._make_file(os.path.join(tmpdir, "sdp", "objects", "1519", "fed")) self._make_file(os.path.join(tmpdir, "sdq", "objects", "9876")) # the empty os.makedirs(os.path.join(tmpdir, "sdr")) os.makedirs(os.path.join(tmpdir, "sds", "objects")) os.makedirs(os.path.join(tmpdir, "sdt", "objects", "9601")) os.makedirs(os.path.join(tmpdir, "sdu", "objects", "6499", "f80")) # the irrelevant os.makedirs(os.path.join(tmpdir, "sdv", "accounts", "77", "421", "4b8c86149a6d532f4af018578fd9f421")) os.makedirs(os.path.join(tmpdir, "sdw", "containers", "28", "51e", "4f9eee668b66c6f0250bfa3c7ab9e51e")) locations = [(loc.path, loc.device, loc.partition) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=False)] locations.sort() self.assertEqual( locations, [(os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "5c1fdc1ffb12e5eaf84edc30d8b67aca"), "sdp", "1519"), (os.path.join(tmpdir, "sdp", "objects", "1519", "aca", "fdfd184d39080020bc8b487f8a7beaca"), "sdp", "1519"), (os.path.join(tmpdir, "sdp", "objects", "1519", "df2", "b0fe7af831cc7b1af5bf486b1c841df2"), "sdp", "1519"), (os.path.join(tmpdir, "sdp", "objects", "9720", "ca5", "4a943bc72c2e647c4675923d58cf4ca5"), "sdp", "9720"), (os.path.join(tmpdir, "sdq", "objects", "3071", "8eb", "fcd938702024c25fef6c32fef05298eb"), "sdq", "3071")]) def test_skipping_unmounted_devices(self): def mock_ismount(path): return path.endswith('sdp') with mock.patch('swift.obj.diskfile.ismount', mock_ismount): with temptree([]) as tmpdir: os.makedirs(os.path.join(tmpdir, "sdp", "objects", "2607", "df3", "ec2871fe724411f91787462f97d30df3")) os.makedirs(os.path.join(tmpdir, "sdq", "objects", "9785", "a10", "4993d582f41be9771505a8d4cb237a10")) locations = [ (loc.path, loc.device, loc.partition) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=True)] locations.sort() self.assertEqual( locations, [(os.path.join(tmpdir, "sdp", "objects", "2607", "df3", "ec2871fe724411f91787462f97d30df3"), "sdp", "2607")]) # Do it again, this time with a logger. ml = mock.MagicMock() locations = [ (loc.path, loc.device, loc.partition) for loc in diskfile.object_audit_location_generator( devices=tmpdir, mount_check=True, logger=ml)] ml.debug.assert_called_once_with( 'Skipping %s as it is not mounted', 'sdq') def test_only_catch_expected_errors(self): # Crazy exceptions should still escape object_audit_location_generator # so that errors get logged and a human can see what's going wrong; # only normal FS corruption should be skipped over silently. def list_locations(dirname): return [(loc.path, loc.device, loc.partition) for loc in diskfile.object_audit_location_generator( devices=dirname, mount_check=False)] real_listdir = os.listdir def splode_if_endswith(suffix): def sploder(path): if path.endswith(suffix): raise OSError(errno.EACCES, "don't try to ad-lib") else: return real_listdir(path) return sploder with temptree([]) as tmpdir: os.makedirs(os.path.join(tmpdir, "sdf", "objects", "2607", "b54", "fe450ec990a88cc4b252b181bab04b54")) with mock.patch('os.listdir', splode_if_endswith("sdf/objects")): self.assertRaises(OSError, list_locations, tmpdir) with mock.patch('os.listdir', splode_if_endswith("2607")): self.assertRaises(OSError, list_locations, tmpdir) with mock.patch('os.listdir', splode_if_endswith("b54")): self.assertRaises(OSError, list_locations, tmpdir) class TestDiskFileManager(unittest.TestCase): def setUp(self): self.tmpdir = mkdtemp() self.testdir = os.path.join( self.tmpdir, 'tmp_test_obj_server_DiskFile') self.existing_device1 = 'sda1' self.existing_device2 = 'sda2' mkdirs(os.path.join(self.testdir, self.existing_device1, 'tmp')) mkdirs(os.path.join(self.testdir, self.existing_device2, 'tmp')) self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) self.conf = dict(devices=self.testdir, mount_check='false', keep_cache_size=2 * 1024) self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) def tearDown(self): rmtree(self.tmpdir, ignore_errors=1) def test_construct_dev_path(self): res_path = self.df_mgr.construct_dev_path('abc') self.assertEqual(os.path.join(self.df_mgr.devices, 'abc'), res_path) def test_pickle_async_update(self): self.df_mgr.logger.increment = mock.MagicMock() ts = normalize_timestamp(10000.0) with mock.patch('swift.obj.diskfile.write_pickle') as wp: self.df_mgr.pickle_async_update(self.existing_device1, 'a', 'c', 'o', dict(a=1, b=2), ts) dp = self.df_mgr.construct_dev_path(self.existing_device1) ohash = diskfile.hash_path('a', 'c', 'o') wp.assert_called_with({'a': 1, 'b': 2}, os.path.join(dp, diskfile.ASYNCDIR, ohash[-3:], ohash + '-' + ts), os.path.join(dp, 'tmp')) self.df_mgr.logger.increment.assert_called_with('async_pendings') def test_object_audit_location_generator(self): locations = list(self.df_mgr.object_audit_location_generator()) self.assertEqual(locations, []) def test_get_hashes_bad_dev(self): self.df_mgr.mount_check = True with mock.patch('swift.obj.diskfile.check_mount', mock.MagicMock(side_effect=[False])): self.assertRaises(DiskFileDeviceUnavailable, self.df_mgr.get_hashes, 'sdb1', '0', '123') def test_get_hashes_w_nothing(self): hashes = self.df_mgr.get_hashes(self.existing_device1, '0', '123') self.assertEqual(hashes, {}) # get_hashes creates the partition path, so call again for code # path coverage, ensuring the result is unchanged hashes = self.df_mgr.get_hashes(self.existing_device1, '0', '123') self.assertEqual(hashes, {}) def test_replication_lock_on(self): # Double check settings self.df_mgr.replication_one_per_device = True self.df_mgr.replication_lock_timeout = 0.1 dev_path = os.path.join(self.testdir, self.existing_device1) with self.df_mgr.replication_lock(dev_path): lock_exc = None exc = None try: with self.df_mgr.replication_lock(dev_path): raise Exception( '%r was not replication locked!' % dev_path) except ReplicationLockTimeout as err: lock_exc = err except Exception as err: exc = err self.assertTrue(lock_exc is not None) self.assertTrue(exc is None) def test_replication_lock_off(self): # Double check settings self.df_mgr.replication_one_per_device = False self.df_mgr.replication_lock_timeout = 0.1 dev_path = os.path.join(self.testdir, self.existing_device1) with self.df_mgr.replication_lock(dev_path): lock_exc = None exc = None try: with self.df_mgr.replication_lock(dev_path): raise Exception( '%r was not replication locked!' % dev_path) except ReplicationLockTimeout as err: lock_exc = err except Exception as err: exc = err self.assertTrue(lock_exc is None) self.assertTrue(exc is not None) def test_replication_lock_another_device_fine(self): # Double check settings self.df_mgr.replication_one_per_device = True self.df_mgr.replication_lock_timeout = 0.1 dev_path = os.path.join(self.testdir, self.existing_device1) dev_path2 = os.path.join(self.testdir, self.existing_device2) with self.df_mgr.replication_lock(dev_path): lock_exc = None try: with self.df_mgr.replication_lock(dev_path2): pass except ReplicationLockTimeout as err: lock_exc = err self.assertTrue(lock_exc is None) class TestDiskFile(unittest.TestCase): """Test swift.obj.diskfile.DiskFile""" def setUp(self): """Set up for testing swift.obj.diskfile""" self.tmpdir = mkdtemp() self.testdir = os.path.join( self.tmpdir, 'tmp_test_obj_server_DiskFile') self.existing_device = 'sda1' mkdirs(os.path.join(self.testdir, self.existing_device, 'tmp')) self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) self.conf = dict(devices=self.testdir, mount_check='false', keep_cache_size=2 * 1024, mb_per_sync=1) self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) def tearDown(self): """Tear down for testing swift.obj.diskfile""" rmtree(self.tmpdir, ignore_errors=1) tpool.execute = self._orig_tpool_exc def _create_ondisk_file(self, df, data, timestamp, metadata=None, ext='.data'): mkdirs(df._datadir) if timestamp is None: timestamp = time() timestamp = normalize_timestamp(timestamp) if not metadata: metadata = {} if 'X-Timestamp' not in metadata: metadata['X-Timestamp'] = normalize_timestamp(timestamp) if 'ETag' not in metadata: etag = md5() etag.update(data) metadata['ETag'] = etag.hexdigest() if 'name' not in metadata: metadata['name'] = '/a/c/o' if 'Content-Length' not in metadata: metadata['Content-Length'] = str(len(data)) data_file = os.path.join(df._datadir, timestamp + ext) with open(data_file, 'wb') as f: f.write(data) xattr.setxattr(f.fileno(), diskfile.METADATA_KEY, pickle.dumps(metadata, diskfile.PICKLE_PROTOCOL)) def _simple_get_diskfile(self, partition='0', account='a', container='c', obj='o'): return self.df_mgr.get_diskfile(self.existing_device, partition, account, container, obj) def _create_test_file(self, data, timestamp=None, metadata=None, account='a', container='c', obj='o'): if metadata is None: metadata = {} metadata.setdefault('name', '/%s/%s/%s' % (account, container, obj)) df = self._simple_get_diskfile(account=account, container=container, obj=obj) self._create_ondisk_file(df, data, timestamp, metadata) df = self._simple_get_diskfile(account=account, container=container, obj=obj) df.open() return df def test_open_not_exist(self): df = self._simple_get_diskfile() self.assertRaises(DiskFileNotExist, df.open) def test_open_expired(self): self.assertRaises(DiskFileExpired, self._create_test_file, '1234567890', metadata={'X-Delete-At': '0'}) def test_open_not_expired(self): try: self._create_test_file( '1234567890', metadata={'X-Delete-At': str(2 * int(time()))}) except SwiftException as err: self.fail("Unexpected swift exception raised: %r" % err) def test_get_metadata(self): df = self._create_test_file('1234567890', timestamp=42) md = df.get_metadata() self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) def test_read_metadata(self): self._create_test_file('1234567890', timestamp=42) df = self._simple_get_diskfile() md = df.read_metadata() self.assertEqual(md['X-Timestamp'], normalize_timestamp(42)) def test_get_metadata_not_opened(self): df = self._simple_get_diskfile() self.assertRaises(DiskFileNotOpen, df.get_metadata) def test_not_opened(self): df = self._simple_get_diskfile() try: with df: pass except DiskFileNotOpen: pass else: self.fail("Expected DiskFileNotOpen exception") def test_disk_file_default_disallowed_metadata(self): # build an object with some meta (ts 41) orig_metadata = {'X-Object-Meta-Key1': 'Value1', 'Content-Type': 'text/garbage'} df = self._get_open_disk_file(ts=41, extra_metadata=orig_metadata) with df.open(): self.assertEquals('1024', df._metadata['Content-Length']) # write some new metadata (fast POST, don't send orig meta, ts 42) df = self._simple_get_diskfile() df.write_metadata({'X-Timestamp': normalize_timestamp(42), 'X-Object-Meta-Key2': 'Value2'}) df = self._simple_get_diskfile() with df.open(): # non-fast-post updateable keys are preserved self.assertEquals('text/garbage', df._metadata['Content-Type']) # original fast-post updateable keys are removed self.assert_('X-Object-Meta-Key1' not in df._metadata) # new fast-post updateable keys are added self.assertEquals('Value2', df._metadata['X-Object-Meta-Key2']) def test_disk_file_reader_iter(self): df = self._create_test_file('1234567890') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) self.assertEqual(''.join(reader), '1234567890') self.assertEqual(quarantine_msgs, []) def test_disk_file_reader_iter_w_quarantine(self): df = self._create_test_file('1234567890') def raise_dfq(m): raise DiskFileQuarantined(m) reader = df.reader(_quarantine_hook=raise_dfq) reader._obj_size += 1 self.assertRaises(DiskFileQuarantined, ''.join, reader) def test_disk_file_app_iter_corners(self): df = self._create_test_file('1234567890') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) self.assertEquals(''.join(reader.app_iter_range(0, None)), '1234567890') self.assertEquals(quarantine_msgs, []) df = self._simple_get_diskfile() with df.open(): reader = df.reader() self.assertEqual(''.join(reader.app_iter_range(5, None)), '67890') def test_disk_file_app_iter_range_w_none(self): df = self._create_test_file('1234567890') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) self.assertEqual(''.join(reader.app_iter_range(None, None)), '1234567890') self.assertEqual(quarantine_msgs, []) def test_disk_file_app_iter_partial_closes(self): df = self._create_test_file('1234567890') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_range(0, 5) self.assertEqual(''.join(it), '12345') self.assertEqual(quarantine_msgs, []) self.assertTrue(reader._fp is None) def test_disk_file_app_iter_ranges(self): df = self._create_test_file('012345678911234567892123456789') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_ranges([(0, 10), (10, 20), (20, 30)], 'plain/text', '\r\n--someheader\r\n', 30) value = ''.join(it) self.assertTrue('0123456789' in value) self.assertTrue('1123456789' in value) self.assertTrue('2123456789' in value) self.assertEqual(quarantine_msgs, []) def test_disk_file_app_iter_ranges_w_quarantine(self): df = self._create_test_file('012345678911234567892123456789') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) reader._obj_size += 1 it = reader.app_iter_ranges([(0, 30)], 'plain/text', '\r\n--someheader\r\n', 30) value = ''.join(it) self.assertTrue('0123456789' in value) self.assertTrue('1123456789' in value) self.assertTrue('2123456789' in value) self.assertEqual(quarantine_msgs, ["Bytes read: 30, does not match metadata: 31"]) def test_disk_file_app_iter_ranges_w_no_etag_quarantine(self): df = self._create_test_file('012345678911234567892123456789') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_ranges([(0, 10)], 'plain/text', '\r\n--someheader\r\n', 30) value = ''.join(it) self.assertTrue('0123456789' in value) self.assertEqual(quarantine_msgs, []) def test_disk_file_app_iter_ranges_edges(self): df = self._create_test_file('012345678911234567892123456789') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_ranges([(3, 10), (0, 2)], 'application/whatever', '\r\n--someheader\r\n', 30) value = ''.join(it) self.assertTrue('3456789' in value) self.assertTrue('01' in value) self.assertEqual(quarantine_msgs, []) def test_disk_file_large_app_iter_ranges(self): # This test case is to make sure that the disk file app_iter_ranges # method all the paths being tested. long_str = '01234567890' * 65536 target_strs = ['3456789', long_str[0:65590]] df = self._create_test_file(long_str) quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_ranges([(3, 10), (0, 65590)], 'plain/text', '5e816ff8b8b8e9a5d355497e5d9e0301', 655360) # The produced string actually missing the MIME headers # need to add these headers to make it as real MIME message. # The body of the message is produced by method app_iter_ranges # off of DiskFile object. header = ''.join(['Content-Type: multipart/byteranges;', 'boundary=', '5e816ff8b8b8e9a5d355497e5d9e0301\r\n']) value = header + ''.join(it) self.assertEquals(quarantine_msgs, []) parts = map(lambda p: p.get_payload(decode=True), email.message_from_string(value).walk())[1:3] self.assertEqual(parts, target_strs) def test_disk_file_app_iter_ranges_empty(self): # This test case tests when empty value passed into app_iter_ranges # When ranges passed into the method is either empty array or None, # this method will yield empty string df = self._create_test_file('012345678911234567892123456789') quarantine_msgs = [] reader = df.reader(_quarantine_hook=quarantine_msgs.append) it = reader.app_iter_ranges([], 'application/whatever', '\r\n--someheader\r\n', 100) self.assertEqual(''.join(it), '') df = self._simple_get_diskfile() with df.open(): reader = df.reader() it = reader.app_iter_ranges(None, 'app/something', '\r\n--someheader\r\n', 150) self.assertEqual(''.join(it), '') self.assertEqual(quarantine_msgs, []) def test_disk_file_mkstemp_creates_dir(self): tmpdir = os.path.join(self.testdir, self.existing_device, 'tmp') os.rmdir(tmpdir) df = self._simple_get_diskfile() with df.create(): self.assert_(os.path.exists(tmpdir)) def _get_open_disk_file(self, invalid_type=None, obj_name='o', fsize=1024, csize=8, mark_deleted=False, prealloc=False, ts=None, mount_check=False, extra_metadata=None): '''returns a DiskFile''' df = self._simple_get_diskfile(obj=obj_name) data = '0' * fsize etag = md5() if ts: timestamp = ts else: timestamp = normalize_timestamp(time()) if prealloc: prealloc_size = fsize else: prealloc_size = None with df.create(size=prealloc_size) as writer: upload_size = writer.write(data) etag.update(data) etag = etag.hexdigest() metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(upload_size), } metadata.update(extra_metadata or {}) writer.put(metadata) if invalid_type == 'ETag': etag = md5() etag.update('1' + '0' * (fsize - 1)) etag = etag.hexdigest() metadata['ETag'] = etag diskfile.write_metadata(writer._fd, metadata) elif invalid_type == 'Content-Length': metadata['Content-Length'] = fsize - 1 diskfile.write_metadata(writer._fd, metadata) elif invalid_type == 'Bad-Content-Length': metadata['Content-Length'] = 'zero' diskfile.write_metadata(writer._fd, metadata) elif invalid_type == 'Missing-Content-Length': del metadata['Content-Length'] diskfile.write_metadata(writer._fd, metadata) elif invalid_type == 'Bad-X-Delete-At': metadata['X-Delete-At'] = 'bad integer' diskfile.write_metadata(writer._fd, metadata) if mark_deleted: df.delete(timestamp) data_files = [os.path.join(df._datadir, fname) for fname in sorted(os.listdir(df._datadir), reverse=True) if fname.endswith('.data')] if invalid_type == 'Corrupt-Xattrs': # We have to go below read_metadata/write_metadata to get proper # corruption. meta_xattr = xattr.getxattr(data_files[0], "user.swift.metadata") wrong_byte = 'X' if meta_xattr[0] != 'X' else 'Y' xattr.setxattr(data_files[0], "user.swift.metadata", wrong_byte + meta_xattr[1:]) elif invalid_type == 'Truncated-Xattrs': meta_xattr = xattr.getxattr(data_files[0], "user.swift.metadata") xattr.setxattr(data_files[0], "user.swift.metadata", meta_xattr[:-1]) elif invalid_type == 'Missing-Name': md = diskfile.read_metadata(data_files[0]) del md['name'] diskfile.write_metadata(data_files[0], md) elif invalid_type == 'Bad-Name': md = diskfile.read_metadata(data_files[0]) md['name'] = md['name'] + 'garbage' diskfile.write_metadata(data_files[0], md) self.conf['disk_chunk_size'] = csize self.conf['mount_check'] = mount_check self.df_mgr = diskfile.DiskFileManager(self.conf, FakeLogger()) df = self._simple_get_diskfile(obj=obj_name) df.open() if invalid_type == 'Zero-Byte': fp = open(df._data_file, 'w') fp.close() df.unit_test_len = fsize return df def test_keep_cache(self): df = self._get_open_disk_file(fsize=65) with mock.patch("swift.obj.diskfile.drop_buffer_cache") as foo: for _ in df.reader(): pass self.assertTrue(foo.called) df = self._get_open_disk_file(fsize=65) with mock.patch("swift.obj.diskfile.drop_buffer_cache") as bar: for _ in df.reader(keep_cache=False): pass self.assertTrue(bar.called) df = self._get_open_disk_file(fsize=65) with mock.patch("swift.obj.diskfile.drop_buffer_cache") as boo: for _ in df.reader(keep_cache=True): pass self.assertFalse(boo.called) df = self._get_open_disk_file(fsize=5 * 1024, csize=256) with mock.patch("swift.obj.diskfile.drop_buffer_cache") as goo: for _ in df.reader(keep_cache=True): pass self.assertTrue(goo.called) def test_quarantine_valids(self): def verify(*args, **kwargs): try: df = self._get_open_disk_file(**kwargs) reader = df.reader() for chunk in reader: pass except DiskFileQuarantined: self.fail( "Unexpected quarantining occurred: args=%r, kwargs=%r" % ( args, kwargs)) else: pass verify(obj_name='1') verify(obj_name='2', csize=1) verify(obj_name='3', csize=100000) def run_quarantine_invalids(self, invalid_type): def verify(*args, **kwargs): open_exc = invalid_type in ('Content-Length', 'Bad-Content-Length', 'Corrupt-Xattrs', 'Truncated-Xattrs', 'Missing-Name', 'Bad-X-Delete-At') open_collision = invalid_type == 'Bad-Name' reader = None quarantine_msgs = [] try: df = self._get_open_disk_file(**kwargs) reader = df.reader(_quarantine_hook=quarantine_msgs.append) except DiskFileQuarantined as err: if not open_exc: self.fail( "Unexpected DiskFileQuarantine raised: %r" % err) return except DiskFileCollision as err: if not open_collision: self.fail( "Unexpected DiskFileCollision raised: %r" % err) return else: if open_exc: self.fail("Expected DiskFileQuarantine exception") try: for chunk in reader: pass except DiskFileQuarantined as err: self.fail("Unexpected DiskFileQuarantine raised: :%r" % err) else: if not open_exc: self.assertEqual(1, len(quarantine_msgs)) verify(invalid_type=invalid_type, obj_name='1') verify(invalid_type=invalid_type, obj_name='2', csize=1) verify(invalid_type=invalid_type, obj_name='3', csize=100000) verify(invalid_type=invalid_type, obj_name='4') def verify_air(params, start=0, adjustment=0): """verify (a)pp (i)ter (r)ange""" open_exc = invalid_type in ('Content-Length', 'Bad-Content-Length', 'Corrupt-Xattrs', 'Truncated-Xattrs', 'Missing-Name', 'Bad-X-Delete-At') open_collision = invalid_type == 'Bad-Name' reader = None try: df = self._get_open_disk_file(**params) reader = df.reader() except DiskFileQuarantined as err: if not open_exc: self.fail( "Unexpected DiskFileQuarantine raised: %r" % err) return except DiskFileCollision as err: if not open_collision: self.fail( "Unexpected DiskFileCollision raised: %r" % err) return else: if open_exc: self.fail("Expected DiskFileQuarantine exception") try: for chunk in reader.app_iter_range( start, df.unit_test_len + adjustment): pass except DiskFileQuarantined as err: self.fail("Unexpected DiskFileQuarantine raised: :%r" % err) verify_air(dict(invalid_type=invalid_type, obj_name='5')) verify_air(dict(invalid_type=invalid_type, obj_name='6'), 0, 100) verify_air(dict(invalid_type=invalid_type, obj_name='7'), 1) verify_air(dict(invalid_type=invalid_type, obj_name='8'), 0, -1) verify_air(dict(invalid_type=invalid_type, obj_name='8'), 1, 1) def test_quarantine_corrupt_xattrs(self): self.run_quarantine_invalids('Corrupt-Xattrs') def test_quarantine_truncated_xattrs(self): self.run_quarantine_invalids('Truncated-Xattrs') def test_quarantine_invalid_etag(self): self.run_quarantine_invalids('ETag') def test_quarantine_invalid_missing_name(self): self.run_quarantine_invalids('Missing-Name') def test_quarantine_invalid_bad_name(self): self.run_quarantine_invalids('Bad-Name') def test_quarantine_invalid_bad_x_delete_at(self): self.run_quarantine_invalids('Bad-X-Delete-At') def test_quarantine_invalid_content_length(self): self.run_quarantine_invalids('Content-Length') def test_quarantine_invalid_content_length_bad(self): self.run_quarantine_invalids('Bad-Content-Length') def test_quarantine_invalid_zero_byte(self): self.run_quarantine_invalids('Zero-Byte') def test_quarantine_deleted_files(self): try: self._get_open_disk_file(invalid_type='Content-Length') except DiskFileQuarantined: pass else: self.fail("Expected DiskFileQuarantined exception") try: self._get_open_disk_file(invalid_type='Content-Length', mark_deleted=True) except DiskFileQuarantined as err: self.fail("Unexpected DiskFileQuarantined exception" " encountered: %r" % err) except DiskFileNotExist: pass else: self.fail("Expected DiskFileNotExist exception") try: self._get_open_disk_file(invalid_type='Content-Length', mark_deleted=True) except DiskFileNotExist: pass else: self.fail("Expected DiskFileNotExist exception") def test_quarantine_missing_content_length(self): self.assertRaises( DiskFileQuarantined, self._get_open_disk_file, invalid_type='Missing-Content-Length') def test_quarantine_bad_content_length(self): self.assertRaises( DiskFileQuarantined, self._get_open_disk_file, invalid_type='Bad-Content-Length') def test_quarantine_fstat_oserror(self): invocations = [0] orig_os_fstat = os.fstat def bad_fstat(fd): invocations[0] += 1 if invocations[0] == 4: # FIXME - yes, this an icky way to get code coverage ... worth # it? raise OSError() return orig_os_fstat(fd) with mock.patch('os.fstat', bad_fstat): self.assertRaises( DiskFileQuarantined, self._get_open_disk_file) def test_quarantine_hashdir_not_a_directory(self): df = self._create_test_file('1234567890', account="abc", container='123', obj='xyz') hashdir = df._datadir rmtree(hashdir) with open(hashdir, 'w'): pass df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', 'xyz') self.assertRaises(DiskFileQuarantined, df.open) # make sure the right thing got quarantined; the suffix dir should not # have moved, as that could have many objects in it self.assertFalse(os.path.exists(hashdir)) self.assertTrue(os.path.exists(os.path.dirname(hashdir))) def test_create_prealloc(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', 'xyz') with mock.patch("swift.obj.diskfile.fallocate") as fa: with df.create(size=200) as writer: used_fd = writer._fd fa.assert_called_with(used_fd, 200) def test_create_prealloc_oserror(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', 'xyz') with mock.patch("swift.obj.diskfile.fallocate", mock.MagicMock(side_effect=OSError( errno.EACCES, os.strerror(errno.EACCES)))): try: with df.create(size=200): pass except DiskFileNoSpace: pass else: self.fail("Expected exception DiskFileNoSpace") def test_create_close_oserror(self): df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', 'xyz') with mock.patch("swift.obj.diskfile.os.close", mock.MagicMock(side_effect=OSError( errno.EACCES, os.strerror(errno.EACCES)))): try: with df.create(size=200): pass except Exception as err: self.fail("Unexpected exception raised: %r" % err) else: pass def test_write_metadata(self): df = self._create_test_file('1234567890') timestamp = normalize_timestamp(time()) metadata = {'X-Timestamp': timestamp, 'X-Object-Meta-test': 'data'} df.write_metadata(metadata) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 2) exp_name = '%s.meta' % timestamp self.assertTrue(exp_name in set(dl)) def test_delete(self): df = self._get_open_disk_file() ts = time() df.delete(ts) exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) def test_open_deleted(self): df = self._get_open_disk_file() ts = time() df.delete(ts) exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) df = self._simple_get_diskfile() self.assertRaises(DiskFileDeleted, df.open) def test_open_deleted_with_corrupt_tombstone(self): df = self._get_open_disk_file() ts = time() df.delete(ts) exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) # it's pickle-format, so removing the last byte is sufficient to # corrupt it ts_fullpath = os.path.join(df._datadir, exp_name) self.assertTrue(os.path.exists(ts_fullpath)) # sanity check meta_xattr = xattr.getxattr(ts_fullpath, "user.swift.metadata") xattr.setxattr(ts_fullpath, "user.swift.metadata", meta_xattr[:-1]) df = self._simple_get_diskfile() self.assertRaises(DiskFileNotExist, df.open) self.assertFalse(os.path.exists(ts_fullpath)) def test_from_audit_location(self): hashdir = self._create_test_file( 'blah blah', account='three', container='blind', obj='mice')._datadir df = self.df_mgr.get_diskfile_from_audit_location( diskfile.AuditLocation(hashdir, self.existing_device, '0')) df.open() self.assertEqual(df._name, '/three/blind/mice') def test_from_audit_location_with_mismatched_hash(self): hashdir = self._create_test_file( 'blah blah', account='this', container='is', obj='right')._datadir datafile = os.path.join(hashdir, os.listdir(hashdir)[0]) meta = diskfile.read_metadata(datafile) meta['name'] = '/this/is/wrong' diskfile.write_metadata(datafile, meta) df = self.df_mgr.get_diskfile_from_audit_location( diskfile.AuditLocation(hashdir, self.existing_device, '0')) self.assertRaises(DiskFileQuarantined, df.open) def test_close_error(self): def mock_handle_close_quarantine(): raise Exception("Bad") df = self._get_open_disk_file(fsize=1024 * 1024 * 2, csize=1024) reader = df.reader() reader._handle_close_quarantine = mock_handle_close_quarantine for chunk in reader: pass # close is called at the end of the iterator self.assertEquals(reader._fp, None) self.assertEquals(len(df._logger.log_dict['error']), 1) def test_mount_checking(self): def _mock_cm(*args, **kwargs): return False with mock.patch("swift.common.constraints.check_mount", _mock_cm): self.assertRaises( DiskFileDeviceUnavailable, self._get_open_disk_file, mount_check=True) def test_ondisk_search_loop_ts_meta_data(self): df = self._simple_get_diskfile() self._create_ondisk_file(df, '', ext='.ts', timestamp=10) self._create_ondisk_file(df, '', ext='.ts', timestamp=9) self._create_ondisk_file(df, '', ext='.meta', timestamp=8) self._create_ondisk_file(df, '', ext='.meta', timestamp=7) self._create_ondisk_file(df, 'B', ext='.data', timestamp=6) self._create_ondisk_file(df, 'A', ext='.data', timestamp=5) df = self._simple_get_diskfile() try: df.open() except DiskFileDeleted as d: self.assertEquals(d.timestamp, normalize_timestamp(10)) else: self.fail("Expected DiskFileDeleted exception") def test_ondisk_search_loop_meta_ts_data(self): df = self._simple_get_diskfile() self._create_ondisk_file(df, '', ext='.meta', timestamp=10) self._create_ondisk_file(df, '', ext='.meta', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, 'B', ext='.data', timestamp=6) self._create_ondisk_file(df, 'A', ext='.data', timestamp=5) df = self._simple_get_diskfile() try: df.open() except DiskFileDeleted as d: self.assertEquals(d.timestamp, normalize_timestamp(8)) else: self.fail("Expected DiskFileDeleted exception") def test_ondisk_search_loop_meta_data_ts(self): df = self._simple_get_diskfile() self._create_ondisk_file(df, '', ext='.meta', timestamp=10) self._create_ondisk_file(df, '', ext='.meta', timestamp=9) self._create_ondisk_file(df, 'B', ext='.data', timestamp=8) self._create_ondisk_file(df, 'A', ext='.data', timestamp=7) self._create_ondisk_file(df, '', ext='.ts', timestamp=6) self._create_ondisk_file(df, '', ext='.ts', timestamp=5) df = self._simple_get_diskfile() with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_data_meta_ts(self): df = self._simple_get_diskfile() self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) self._create_ondisk_file(df, '', ext='.meta', timestamp=5) df = self._simple_get_diskfile() with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_wayward_files_ignored(self): df = self._simple_get_diskfile() self._create_ondisk_file(df, 'X', ext='.bar', timestamp=11) self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) self._create_ondisk_file(df, '', ext='.meta', timestamp=5) df = self._simple_get_diskfile() with df.open(): self.assertTrue('X-Timestamp' in df._metadata) self.assertEquals(df._metadata['X-Timestamp'], normalize_timestamp(10)) self.assertTrue('deleted' not in df._metadata) def test_ondisk_search_loop_listdir_error(self): df = self._simple_get_diskfile() def mock_listdir_exp(*args, **kwargs): raise OSError(errno.EACCES, os.strerror(errno.EACCES)) with mock.patch("os.listdir", mock_listdir_exp): self._create_ondisk_file(df, 'X', ext='.bar', timestamp=11) self._create_ondisk_file(df, 'B', ext='.data', timestamp=10) self._create_ondisk_file(df, 'A', ext='.data', timestamp=9) self._create_ondisk_file(df, '', ext='.ts', timestamp=8) self._create_ondisk_file(df, '', ext='.ts', timestamp=7) self._create_ondisk_file(df, '', ext='.meta', timestamp=6) self._create_ondisk_file(df, '', ext='.meta', timestamp=5) df = self._simple_get_diskfile() self.assertRaises(DiskFileError, df.open) def test_exception_in_handle_close_quarantine(self): df = self._get_open_disk_file() def blow_up(): raise Exception('a very special error') reader = df.reader() reader._handle_close_quarantine = blow_up for _ in reader: pass reader.close() log_lines = df._logger.get_lines_for_level('error') self.assert_('a very special error' in log_lines[-1]) def test_get_diskfile_from_hash_dev_path_fail(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = ['1381679759.90941.data'] readmeta.return_value = {'name': '/a/c/o'} self.assertRaises( DiskFileDeviceUnavailable, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_not_dir(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata'), mock.patch('swift.obj.diskfile.quarantine_renamer')) as \ (dfclass, hclistdir, readmeta, quarantine_renamer): osexc = OSError() osexc.errno = errno.ENOTDIR hclistdir.side_effect = osexc readmeta.return_value = {'name': '/a/c/o'} self.assertRaises( DiskFileNotExist, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') quarantine_renamer.assert_called_once_with( '/srv/dev/', '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_no_dir(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): osexc = OSError() osexc.errno = errno.ENOENT hclistdir.side_effect = osexc readmeta.return_value = {'name': '/a/c/o'} self.assertRaises( DiskFileNotExist, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_other_oserror(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): osexc = OSError() hclistdir.side_effect = osexc readmeta.return_value = {'name': '/a/c/o'} self.assertRaises( OSError, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_no_actual_files(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = [] readmeta.return_value = {'name': '/a/c/o'} self.assertRaises( DiskFileNotExist, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_read_metadata_problem(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = ['1381679759.90941.data'] readmeta.side_effect = EOFError() self.assertRaises( DiskFileNotExist, self.df_mgr.get_diskfile_from_hash, 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') def test_get_diskfile_from_hash_no_meta_name(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = ['1381679759.90941.data'] readmeta.return_value = {} try: self.df_mgr.get_diskfile_from_hash( 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') except DiskFileNotExist as err: exc = err self.assertEqual(str(exc), '') def test_get_diskfile_from_hash_bad_meta_name(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = ['1381679759.90941.data'] readmeta.return_value = {'name': 'bad'} try: self.df_mgr.get_diskfile_from_hash( 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') except DiskFileNotExist as err: exc = err self.assertEqual(str(exc), '') def test_get_diskfile_from_hash(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value='/srv/dev/') with nested( mock.patch('swift.obj.diskfile.DiskFile'), mock.patch('swift.obj.diskfile.hash_cleanup_listdir'), mock.patch('swift.obj.diskfile.read_metadata')) as \ (dfclass, hclistdir, readmeta): hclistdir.return_value = ['1381679759.90941.data'] readmeta.return_value = {'name': '/a/c/o'} self.df_mgr.get_diskfile_from_hash( 'dev', '9', '9a7175077c01a23ade5956b8a2bba900') dfclass.assert_called_once_with( self.df_mgr, '/srv/dev/', self.df_mgr.threadpools['dev'], '9', 'a', 'c', 'o') hclistdir.assert_called_once_with( '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900', 604800) readmeta.assert_called_once_with( '/srv/dev/objects/9/900/9a7175077c01a23ade5956b8a2bba900/' '1381679759.90941.data') def test_listdir_enoent(self): oserror = OSError() oserror.errno = errno.ENOENT self.df_mgr.logger.error = mock.MagicMock() with mock.patch('os.listdir', side_effect=oserror): self.assertEqual(self.df_mgr._listdir('path'), []) self.assertEqual(self.df_mgr.logger.error.mock_calls, []) def test_listdir_other_oserror(self): oserror = OSError() self.df_mgr.logger.error = mock.MagicMock() with mock.patch('os.listdir', side_effect=oserror): self.assertEqual(self.df_mgr._listdir('path'), []) self.df_mgr.logger.error.assert_called_once_with( 'ERROR: Skipping %r due to error with listdir attempt: %s', 'path', oserror) def test_listdir(self): self.df_mgr.logger.error = mock.MagicMock() with mock.patch('os.listdir', return_value=['abc', 'def']): self.assertEqual(self.df_mgr._listdir('path'), ['abc', 'def']) self.assertEqual(self.df_mgr.logger.error.mock_calls, []) def test_yield_suffixes_dev_path_fail(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) exc = None try: list(self.df_mgr.yield_suffixes('dev', '9')) except DiskFileDeviceUnavailable as err: exc = err self.assertEqual(str(exc), '') def test_yield_suffixes(self): self.df_mgr._listdir = mock.MagicMock(return_value=[ 'abc', 'def', 'ghi', 'abcd', '012']) self.assertEqual( list(self.df_mgr.yield_suffixes('dev', '9')), [(self.testdir + '/dev/objects/9/abc', 'abc'), (self.testdir + '/dev/objects/9/def', 'def'), (self.testdir + '/dev/objects/9/012', '012')]) def test_yield_hashes_dev_path_fail(self): self.df_mgr.get_dev_path = mock.MagicMock(return_value=None) exc = None try: list(self.df_mgr.yield_hashes('dev', '9')) except DiskFileDeviceUnavailable as err: exc = err self.assertEqual(str(exc), '') def test_yield_hashes_empty(self): def _listdir(path): return [] with mock.patch('os.listdir', _listdir): self.assertEqual(list(self.df_mgr.yield_hashes('dev', '9')), []) def test_yield_hashes_empty_suffixes(self): def _listdir(path): return [] with mock.patch('os.listdir', _listdir): self.assertEqual( list(self.df_mgr.yield_hashes('dev', '9', suffixes=['456'])), []) def test_yield_hashes(self): fresh_ts = normalize_timestamp(time() - 10) fresher_ts = normalize_timestamp(time() - 1) def _listdir(path): if path.endswith('/dev/objects/9'): return ['abc', '456', 'def'] elif path.endswith('/dev/objects/9/abc'): return ['9373a92d072897b136b3fc06595b4abc'] elif path.endswith( '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc'): return [fresh_ts + '.ts'] elif path.endswith('/dev/objects/9/456'): return ['9373a92d072897b136b3fc06595b0456', '9373a92d072897b136b3fc06595b7456'] elif path.endswith( '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456'): return ['1383180000.12345.data'] elif path.endswith( '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456'): return [fresh_ts + '.ts', fresher_ts + '.data'] elif path.endswith('/dev/objects/9/def'): return [] else: raise Exception('Unexpected listdir of %r' % path) with nested( mock.patch('os.listdir', _listdir), mock.patch('os.unlink')): self.assertEqual( list(self.df_mgr.yield_hashes('dev', '9')), [(self.testdir + '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc', '9373a92d072897b136b3fc06595b4abc', fresh_ts), (self.testdir + '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456', '9373a92d072897b136b3fc06595b0456', '1383180000.12345'), (self.testdir + '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456', '9373a92d072897b136b3fc06595b7456', fresher_ts)]) def test_yield_hashes_suffixes(self): fresh_ts = normalize_timestamp(time() - 10) fresher_ts = normalize_timestamp(time() - 1) def _listdir(path): if path.endswith('/dev/objects/9'): return ['abc', '456', 'def'] elif path.endswith('/dev/objects/9/abc'): return ['9373a92d072897b136b3fc06595b4abc'] elif path.endswith( '/dev/objects/9/abc/9373a92d072897b136b3fc06595b4abc'): return [fresh_ts + '.ts'] elif path.endswith('/dev/objects/9/456'): return ['9373a92d072897b136b3fc06595b0456', '9373a92d072897b136b3fc06595b7456'] elif path.endswith( '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456'): return ['1383180000.12345.data'] elif path.endswith( '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456'): return [fresh_ts + '.ts', fresher_ts + '.data'] elif path.endswith('/dev/objects/9/def'): return [] else: raise Exception('Unexpected listdir of %r' % path) with nested( mock.patch('os.listdir', _listdir), mock.patch('os.unlink')): self.assertEqual( list(self.df_mgr.yield_hashes( 'dev', '9', suffixes=['456'])), [(self.testdir + '/dev/objects/9/456/9373a92d072897b136b3fc06595b0456', '9373a92d072897b136b3fc06595b0456', '1383180000.12345'), (self.testdir + '/dev/objects/9/456/9373a92d072897b136b3fc06595b7456', '9373a92d072897b136b3fc06595b7456', fresher_ts)]) def test_diskfile_names(self): df = self._simple_get_diskfile() self.assertEqual(df.account, 'a') self.assertEqual(df.container, 'c') self.assertEqual(df.obj, 'o') def test_diskfile_content_length_not_open(self): df = self._simple_get_diskfile() exc = None try: df.content_length except DiskFileNotOpen as err: exc = err self.assertEqual(str(exc), '') def test_diskfile_content_length_deleted(self): df = self._get_open_disk_file() ts = time() df.delete(ts) exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) df = self._simple_get_diskfile() exc = None try: with df.open(): df.content_length except DiskFileDeleted as err: exc = err self.assertEqual(str(exc), '') def test_diskfile_content_length(self): self._get_open_disk_file() df = self._simple_get_diskfile() with df.open(): self.assertEqual(df.content_length, 1024) def test_diskfile_timestamp_not_open(self): df = self._simple_get_diskfile() exc = None try: df.timestamp except DiskFileNotOpen as err: exc = err self.assertEqual(str(exc), '') def test_diskfile_timestamp_deleted(self): df = self._get_open_disk_file() ts = time() df.delete(ts) exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 1) self.assertTrue(exp_name in set(dl)) df = self._simple_get_diskfile() exc = None try: with df.open(): df.timestamp except DiskFileDeleted as err: exc = err self.assertEqual(str(exc), '') def test_diskfile_timestamp(self): self._get_open_disk_file(ts='1383181759.12345') df = self._simple_get_diskfile() with df.open(): self.assertEqual(df.timestamp, '1383181759.12345') def test_error_in_hash_cleanup_listdir(self): def mock_hcl(*args, **kwargs): raise OSError() df = self._get_open_disk_file() ts = time() with mock.patch("swift.obj.diskfile.hash_cleanup_listdir", mock_hcl): try: df.delete(ts) except OSError: self.fail("OSError raised when it should have been swallowed") exp_name = '%s.ts' % str(normalize_timestamp(ts)) dl = os.listdir(df._datadir) self.assertEquals(len(dl), 2) self.assertTrue(exp_name in set(dl)) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/obj/__init__.py0000664000175400017540000000000012323703611020674 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/test_locale/0000775000175400017540000000000012323703665020332 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/test_locale/eo/0000775000175400017540000000000012323703665020735 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/test_locale/eo/LC_MESSAGES/0000775000175400017540000000000012323703665022522 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/test_locale/eo/LC_MESSAGES/swift.mo0000777000175400017540000000000012323703611026767 2../../messages.moustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/test_locale/eo.po0000664000175400017540000000005412323703611021263 0ustar jenkinsjenkins00000000000000msgid "test message" msgstr "prova mesaĝo" swift-1.13.1/test/unit/test_locale/README0000664000175400017540000000011212323703611021173 0ustar jenkinsjenkins00000000000000rebuild the .mo with msgfmt (included with GNU gettext) msgfmt eo.po swift-1.13.1/test/unit/test_locale/test_locale.py0000664000175400017540000000517612323703611023202 0ustar jenkinsjenkins00000000000000#!/usr/bin/env python #-*- coding:utf-8 -*- # Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import unittest import string import sys import threading try: from subprocess import check_output except ImportError: from subprocess import Popen, PIPE, CalledProcessError def check_output(*popenargs, **kwargs): """Lifted from python 2.7 stdlib.""" if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be ' 'overridden.') process = Popen(stdout=PIPE, *popenargs, **kwargs) output, unused_err = process.communicate() retcode = process.poll() if retcode: cmd = kwargs.get("args") if cmd is None: cmd = popenargs[0] raise CalledProcessError(retcode, cmd, output=output) return output class TestTranslations(unittest.TestCase): def setUp(self): self.orig_env = {} for var in 'LC_ALL', 'SWIFT_LOCALEDIR', 'LANGUAGE': self.orig_env[var] = os.environ.get(var) os.environ['LC_ALL'] = 'eo' os.environ['SWIFT_LOCALEDIR'] = os.path.dirname(__file__) os.environ['LANGUAGE'] = '' self.orig_stop = threading._DummyThread._Thread__stop # See http://stackoverflow.com/questions/13193278/\ # understand-python-threading-bug threading._DummyThread._Thread__stop = lambda x: 42 def tearDown(self): for var, val in self.orig_env.iteritems(): if val is not None: os.environ[var] = val else: del os.environ[var] threading._DummyThread._Thread__stop = self.orig_stop def test_translations(self): path = ':'.join(sys.path) translated_message = check_output(['python', __file__, path]) self.assertEquals(translated_message, 'prova mesaĝo\n') if __name__ == "__main__": os.environ['LC_ALL'] = 'eo' os.environ['SWIFT_LOCALEDIR'] = os.path.dirname(__file__) sys.path = string.split(sys.argv[1], ':') from swift import gettext_ as _ print _('test message') swift-1.13.1/test/unit/test_locale/messages.mo0000664000175400017540000000012312323703611022461 0ustar jenkinsjenkins00000000000000$, 8 Etest messageprova mesaĝoswift-1.13.1/test/unit/test_locale/__init__.py0000664000175400017540000000000012323703611022420 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/cli/0000775000175400017540000000000012323703665016603 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/cli/test_info.py0000664000175400017540000002617612323703614021155 0ustar jenkinsjenkins00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy # of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Tests for swift.cli.info""" import os import unittest import cPickle as pickle import mock from cStringIO import StringIO from contextlib import closing from gzip import GzipFile from shutil import rmtree from tempfile import mkdtemp from swift.common import ring, utils from swift.common.swob import Request from swift.cli.info import print_db_info_metadata, print_ring_locations, \ print_info, InfoSystemExit from swift.account.server import AccountController from swift.container.server import ContainerController class TestCliInfo(unittest.TestCase): def setUp(self): self.orig_hp = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX utils.HASH_PATH_PREFIX = 'info' utils.HASH_PATH_SUFFIX = 'info' self.testdir = os.path.join(mkdtemp(), 'tmp_test_cli_info') utils.mkdirs(self.testdir) rmtree(self.testdir) utils.mkdirs(os.path.join(self.testdir, 'sda1')) utils.mkdirs(os.path.join(self.testdir, 'sda1', 'tmp')) utils.mkdirs(os.path.join(self.testdir, 'sdb1')) utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp')) self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz') with closing(GzipFile(self.account_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1', 'port': 42}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.2', 'port': 43}], 30), f) self.container_ring_path = os.path.join(self.testdir, 'container.ring.gz') with closing(GzipFile(self.container_ring_path, 'wb')) as f: pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]], [{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.3', 'port': 42}, {'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.4', 'port': 43}], 30), f) def tearDown(self): utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp rmtree(os.path.dirname(self.testdir)) def assertRaisesMessage(self, exc, msg, func, *args, **kwargs): try: func(*args, **kwargs) except Exception, e: self.assertEqual(msg, str(e)) self.assertTrue(isinstance(e, exc), "Expected %s, got %s" % (exc, type(e))) def test_print_db_info_metadata(self): self.assertRaisesMessage(ValueError, 'Wrong DB type', print_db_info_metadata, 't', {}, {}) self.assertRaisesMessage(ValueError, 'DB info is None', print_db_info_metadata, 'container', None, {}) self.assertRaisesMessage(ValueError, 'Info is incomplete', print_db_info_metadata, 'container', {}, {}) info = dict( account='acct', created_at=100.1, put_timestamp=106.3, delete_timestamp=107.9, object_count='20', bytes_used='42') info['hash'] = 'abaddeadbeefcafe' info['id'] = 'abadf100d0ddba11' md = {'x-account-meta-mydata': ('swift', '0000000000.00000'), 'x-other-something': ('boo', '0000000000.00000')} out = StringIO() with mock.patch('sys.stdout', out): print_db_info_metadata('account', info, md) exp_out = '''Path: /acct Account: acct Account Hash: dc5be2aa4347a22a0fee6bc7de505b47 Metadata: Created at: 1970-01-01 00:01:40.100000 (100.1) Put Timestamp: 1970-01-01 00:01:46.300000 (106.3) Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9) Object Count: 20 Bytes Used: 42 Chexor: abaddeadbeefcafe UUID: abadf100d0ddba11 X-Other-Something: boo No system metadata found in db file User Metadata: {'mydata': 'swift'}''' self.assertEquals(out.getvalue().strip(), exp_out) info = dict( account='acct', container='cont', created_at='0000000100.10000', put_timestamp='0000000106.30000', delete_timestamp='0000000107.90000', object_count='20', bytes_used='42', reported_put_timestamp='0000010106.30000', reported_delete_timestamp='0000010107.90000', reported_object_count='20', reported_bytes_used='42', x_container_foo='bar', x_container_bar='goo') info['hash'] = 'abaddeadbeefcafe' info['id'] = 'abadf100d0ddba11' md = {'x-container-sysmeta-mydata': ('swift', '0000000000.00000')} out = StringIO() with mock.patch('sys.stdout', out): print_db_info_metadata('container', info, md) exp_out = '''Path: /acct/cont Account: acct Container: cont Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae Metadata: Created at: 1970-01-01 00:01:40.100000 (0000000100.10000) Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000) Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000) Object Count: 20 Bytes Used: 42 Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000) Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000) Reported Object Count: 20 Reported Bytes Used: 42 Chexor: abaddeadbeefcafe UUID: abadf100d0ddba11 X-Container-Bar: goo X-Container-Foo: bar System Metadata: {'mydata': 'swift'} No user metadata found in db file''' self.assertEquals(out.getvalue().strip(), exp_out) def test_print_ring_locations(self): self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, None, 'dir', 'acct') self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, [], None, 'acct') self.assertRaisesMessage(ValueError, 'None type', print_ring_locations, [], 'dir', None) self.assertRaisesMessage(ValueError, 'Ring error', print_ring_locations, [], 'dir', 'acct', 'con') out = StringIO() with mock.patch('sys.stdout', out): acctring = ring.Ring(self.testdir, ring_name='account') print_ring_locations(acctring, 'dir', 'acct') exp_db2 = os.path.join('/srv', 'node', 'sdb1', 'dir', '3', 'b47', 'dc5be2aa4347a22a0fee6bc7de505b47', 'dc5be2aa4347a22a0fee6bc7de505b47.db') exp_db1 = os.path.join('/srv', 'node', 'sda1', 'dir', '3', 'b47', 'dc5be2aa4347a22a0fee6bc7de505b47', 'dc5be2aa4347a22a0fee6bc7de505b47.db') exp_out = ('Ring locations:\n 127.0.0.2:43 - %s\n' ' 127.0.0.1:42 - %s\n' '\nnote: /srv/node is used as default value of `devices`,' ' the real value is set in the account config file on' ' each storage node.' % (exp_db2, exp_db1)) self.assertEquals(out.getvalue().strip(), exp_out) out = StringIO() with mock.patch('sys.stdout', out): contring = ring.Ring(self.testdir, ring_name='container') print_ring_locations(contring, 'dir', 'acct', 'con') exp_db4 = os.path.join('/srv', 'node', 'sdb1', 'dir', '1', 'fe6', '63e70955d78dfc62821edc07d6ec1fe6', '63e70955d78dfc62821edc07d6ec1fe6.db') exp_db3 = os.path.join('/srv', 'node', 'sda1', 'dir', '1', 'fe6', '63e70955d78dfc62821edc07d6ec1fe6', '63e70955d78dfc62821edc07d6ec1fe6.db') exp_out = ('Ring locations:\n 127.0.0.4:43 - %s\n' ' 127.0.0.3:42 - %s\n' '\nnote: /srv/node is used as default value of `devices`,' ' the real value is set in the container config file on' ' each storage node.' % (exp_db4, exp_db3)) self.assertEquals(out.getvalue().strip(), exp_out) def test_print_info(self): db_file = 'foo' self.assertRaises(InfoSystemExit, print_info, 'object', db_file) db_file = os.path.join(self.testdir, './acct.db') self.assertRaises(InfoSystemExit, print_info, 'account', db_file) controller = AccountController( {'devices': self.testdir, 'mount_check': 'false'}) req = Request.blank('/sda1/1/acct', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(controller) self.assertEqual(resp.status_int, 201) out = StringIO() exp_raised = False with mock.patch('sys.stdout', out): db_file = os.path.join(self.testdir, 'sda1', 'accounts', '1', 'b47', 'dc5be2aa4347a22a0fee6bc7de505b47', 'dc5be2aa4347a22a0fee6bc7de505b47.db') try: print_info('account', db_file, swift_dir=self.testdir) except Exception: exp_raised = True if exp_raised: self.fail("Unexpected exception raised") else: self.assertTrue(len(out.getvalue().strip()) > 800) controller = ContainerController( {'devices': self.testdir, 'mount_check': 'false'}) req = Request.blank('/sda1/1/acct/cont', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) resp = req.get_response(controller) self.assertEqual(resp.status_int, 201) out = StringIO() exp_raised = False with mock.patch('sys.stdout', out): db_file = os.path.join(self.testdir, 'sda1', 'containers', '1', 'cae', 'd49d0ecbb53be1fcc49624f2f7c7ccae', 'd49d0ecbb53be1fcc49624f2f7c7ccae.db') orig_cwd = os.getcwd() try: os.chdir(os.path.dirname(db_file)) print_info('container', os.path.basename(db_file), swift_dir='/dev/null') except Exception: exp_raised = True finally: os.chdir(orig_cwd) if exp_raised: self.fail("Unexpected exception raised") else: self.assertTrue(len(out.getvalue().strip()) > 600) swift-1.13.1/test/unit/cli/test_recon.py0000664000175400017540000001344012323703611021313 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 Christian Schwede # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import json import mock import os import random import string import tempfile import time import unittest from eventlet.green import urllib2 from swift.cli import recon from swift.common import utils from swift.common.ring import builder class TestHelpers(unittest.TestCase): def test_seconds2timeunit(self): self.assertEqual(recon.seconds2timeunit(10), (10, 'seconds')) self.assertEqual(recon.seconds2timeunit(600), (10, 'minutes')) self.assertEqual(recon.seconds2timeunit(36000), (10, 'hours')) self.assertEqual(recon.seconds2timeunit(60 * 60 * 24 * 10), (10, 'days')) def test_size_suffix(self): self.assertEqual(recon.size_suffix(5 * 10 ** 2), '500 bytes') self.assertEqual(recon.size_suffix(5 * 10 ** 3), '5 kB') self.assertEqual(recon.size_suffix(5 * 10 ** 6), '5 MB') self.assertEqual(recon.size_suffix(5 * 10 ** 9), '5 GB') self.assertEqual(recon.size_suffix(5 * 10 ** 12), '5 TB') self.assertEqual(recon.size_suffix(5 * 10 ** 15), '5 PB') self.assertEqual(recon.size_suffix(5 * 10 ** 18), '5 EB') self.assertEqual(recon.size_suffix(5 * 10 ** 21), '5 ZB') class TestScout(unittest.TestCase): def setUp(self, *_args, **_kwargs): self.scout_instance = recon.Scout("type", suppress_errors=True) self.url = 'http://127.0.0.1:8080/recon/type' @mock.patch('eventlet.green.urllib2.urlopen') def test_scout_ok(self, mock_urlopen): mock_urlopen.return_value.read = lambda: json.dumps([]) url, content, status = self.scout_instance.scout( ("127.0.0.1", "8080")) self.assertEqual(url, self.url) self.assertEqual(content, []) self.assertEqual(status, 200) @mock.patch('eventlet.green.urllib2.urlopen') def test_scout_url_error(self, mock_urlopen): mock_urlopen.side_effect = urllib2.URLError("") url, content, status = self.scout_instance.scout( ("127.0.0.1", "8080")) self.assertTrue(isinstance(content, urllib2.URLError)) self.assertEqual(url, self.url) self.assertEqual(status, -1) @mock.patch('eventlet.green.urllib2.urlopen') def test_scout_http_error(self, mock_urlopen): mock_urlopen.side_effect = urllib2.HTTPError( self.url, 404, "Internal error", None, None) url, content, status = self.scout_instance.scout( ("127.0.0.1", "8080")) self.assertEqual(url, self.url) self.assertTrue(isinstance(content, urllib2.HTTPError)) self.assertEqual(status, 404) class TestRecon(unittest.TestCase): def setUp(self, *_args, **_kwargs): self.recon_instance = recon.SwiftRecon() self.swift_dir = tempfile.gettempdir() self.ring_name = "test_object_%s" % ( ''.join(random.choice(string.digits) for x in range(6))) self.tmpfile_name = "%s/%s.ring.gz" % (self.swift_dir, self.ring_name) utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_PREFIX = 'startcap' def tearDown(self, *_args, **_kwargs): try: os.remove(self.tmpfile_name) except: pass def test_gen_stats(self): stats = self.recon_instance._gen_stats((1, 4, 10, None), 'Sample') self.assertEqual(stats.get('name'), 'Sample') self.assertEqual(stats.get('average'), 5.0) self.assertEqual(stats.get('high'), 10) self.assertEqual(stats.get('reported'), 3) self.assertEqual(stats.get('low'), 1) self.assertEqual(stats.get('total'), 15) self.assertEqual(stats.get('number_none'), 1) self.assertEqual(stats.get('perc_none'), 25.0) def test_ptime(self): with mock.patch('time.localtime') as mock_localtime: mock_localtime.return_value = time.struct_time( (2013, 12, 17, 10, 0, 0, 1, 351, 0)) timestamp = self.recon_instance._ptime(1387274400) self.assertEqual(timestamp, "2013-12-17 10:00:00") mock_localtime.assertCalledWith(1387274400) timestamp2 = self.recon_instance._ptime() self.assertEqual(timestamp2, "2013-12-17 10:00:00") mock_localtime.assertCalledWith() def test_get_devices(self): ringbuilder = builder.RingBuilder(2, 3, 1) ringbuilder.add_dev({'id': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1', 'region': 0}) ringbuilder.add_dev({'id': 1, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1', 'region': 0}) ringbuilder.rebalance() ringbuilder.get_ring().save(self.tmpfile_name) ips = self.recon_instance.get_devices( None, self.swift_dir, self.ring_name) self.assertEqual( set([('127.0.0.1', 10000), ('127.0.0.1', 10001)]), ips) ips = self.recon_instance.get_devices( 0, self.swift_dir, self.ring_name) self.assertEqual(set([('127.0.0.1', 10000)]), ips) ips = self.recon_instance.get_devices( 1, self.swift_dir, self.ring_name) self.assertEqual(set([('127.0.0.1', 10001)]), ips) swift-1.13.1/test/unit/cli/test_ringbuilder.py0000664000175400017540000001753712323703611022526 0ustar jenkinsjenkins00000000000000# Copyright (c) 2014 Christian Schwede # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import tempfile import unittest import swift.cli.ringbuilder from swift.common.ring import RingBuilder class TestCommands(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestCommands, self).__init__(*args, **kwargs) # List of search values for various actions # These should all match the first device in the sample ring # (see below) but not the second device self.search_values = ["d0", "/sda1", "r0", "z0", "z0-127.0.0.1", "127.0.0.1", "z0:6000", ":6000", "R127.0.0.1", "127.0.0.1R127.0.0.1", "R:6000", "_some meta data"] tmpf = tempfile.NamedTemporaryFile() self.tmpfile = tmpf.name def tearDown(self): try: os.remove(self.tmpfile) except OSError: pass def create_sample_ring(self): """ Create a sample ring with two devices At least two devices are needed to test removing a device, since removing the last device of a ring is not allowed """ # Ensure there is no existing test builder file because # create_sample_ring() might be used more than once in a single test try: os.remove(self.tmpfile) except OSError: pass ring = RingBuilder(6, 3, 1) ring.add_dev({'weight': 100.0, 'region': 0, 'zone': 0, 'ip': '127.0.0.1', 'port': 6000, 'device': 'sda1', 'meta': 'some meta data', }) ring.add_dev({'weight': 100.0, 'region': 1, 'zone': 1, 'ip': '127.0.0.2', 'port': 6001, 'device': 'sda2' }) ring.save(self.tmpfile) def test_create_ring(self): argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.part_power, 6) self.assertEqual(ring.replicas, 3.14159265359) self.assertEqual(ring.min_part_hours, 1) def test_add_device(self): self.create_sample_ring() argv = ["", self.tmpfile, "add", "r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) dev = [d for d in ring.devs if d['id'] == 2][0] self.assertEqual(dev['region'], 2) self.assertEqual(dev['zone'], 3) self.assertEqual(dev['ip'], '127.0.0.1') self.assertEqual(dev['port'], 6000) self.assertEqual(dev['device'], 'sda3') self.assertEqual(dev['weight'], 3.14159265359) self.assertEqual(dev['replication_ip'], '127.0.0.1') self.assertEqual(dev['replication_port'], 6000) self.assertEqual(dev['meta'], 'some meta data') # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate()) def test_remove_device(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "remove", search_value] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was set to 0 dev = [d for d in ring.devs if d['id'] == 0][0] self.assertEqual(dev['weight'], 0) # Check that device is in list of devices to be removed dev = [d for d in ring._remove_devs if d['id'] == 0][0] self.assertEqual(dev['region'], 0) self.assertEqual(dev['zone'], 0) self.assertEqual(dev['ip'], '127.0.0.1') self.assertEqual(dev['port'], 6000) self.assertEqual(dev['device'], 'sda1') self.assertEqual(dev['weight'], 0) self.assertEqual(dev['replication_ip'], '127.0.0.1') self.assertEqual(dev['replication_port'], 6000) self.assertEqual(dev['meta'], 'some meta data') # Check that second device in ring is not affected dev = [d for d in ring.devs if d['id'] == 1][0] self.assertEqual(dev['weight'], 100) self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate()) def test_set_weight(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_weight", search_value, "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was changed dev = [d for d in ring.devs if d['id'] == 0][0] self.assertEqual(dev['weight'], 3.14159265359) # Check that second device in ring is not affected dev = [d for d in ring.devs if d['id'] == 1][0] self.assertEqual(dev['weight'], 100) # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate()) def test_set_info(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_info", search_value, "127.0.1.1:8000/sda1_other meta data"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) dev = [d for d in ring.devs if d['id'] == 0][0] self.assertEqual(dev['ip'], '127.0.1.1') self.assertEqual(dev['port'], 8000) self.assertEqual(dev['device'], 'sda1') self.assertEqual(dev['meta'], 'other meta data') # Check that second device in ring is not affected dev = [d for d in ring.devs if d['id'] == 1][0] self.assertEqual(dev['ip'], '127.0.0.2') self.assertEqual(dev['port'], 6001) self.assertEqual(dev['device'], 'sda2') self.assertEqual(dev['meta'], '') # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate()) def test_set_min_part_hours(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_min_part_hours", "24"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.min_part_hours, 24) def test_set_replicas(self): self.create_sample_ring() argv = ["", self.tmpfile, "set_replicas", "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.replicas, 3.14159265359) if __name__ == '__main__': unittest.main() swift-1.13.1/test/unit/cli/__init__.py0000664000175400017540000000000012323703611020671 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/unit/__init__.py0000664000175400017540000004106712323703611020144 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Swift tests """ import os import copy import logging import errno import sys from contextlib import contextmanager from collections import defaultdict from tempfile import NamedTemporaryFile import time from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree from test import get_config from swift.common.utils import config_true_value, LogAdapter from hashlib import md5 from eventlet import sleep, Timeout import logging.handlers from httplib import HTTPException from numbers import Number class FakeRing(object): def __init__(self, replicas=3, max_more_nodes=0): # 9 total nodes (6 more past the initial 3) is the cap, no matter if # this is set higher, or R^2 for R replicas self.replicas = replicas self.max_more_nodes = max_more_nodes self.devs = {} def set_replicas(self, replicas): self.replicas = replicas self.devs = {} @property def replica_count(self): return self.replicas def get_part(self, account, container=None, obj=None): return 1 def get_nodes(self, account, container=None, obj=None): devs = [] for x in xrange(self.replicas): devs.append(self.devs.get(x)) if devs[x] is None: self.devs[x] = devs[x] = \ {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sd' + (chr(ord('a') + x)), 'zone': x % 3, 'region': x % 2, 'id': x} return 1, devs def get_part_nodes(self, part): return self.get_nodes('blah')[1] def get_more_nodes(self, part): # replicas^2 is the true cap for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes, self.replicas * self.replicas)): yield {'ip': '10.0.0.%s' % x, 'port': 1000 + x, 'device': 'sda', 'zone': x % 3, 'region': x % 2, 'id': x} class FakeMemcache(object): def __init__(self): self.store = {} def get(self, key): return self.store.get(key) def keys(self): return self.store.keys() def set(self, key, value, time=0): self.store[key] = value return True def incr(self, key, time=0): self.store[key] = self.store.setdefault(key, 0) + 1 return self.store[key] @contextmanager def soft_lock(self, key, timeout=0, retries=5): yield True def delete(self, key): try: del self.store[key] except Exception: pass return True def readuntil2crlfs(fd): rv = '' lc = '' crlfs = 0 while crlfs < 2: c = fd.read(1) if not c: raise ValueError("didn't get two CRLFs; just got %r" % rv) rv = rv + c if c == '\r' and lc != '\n': crlfs = 0 if lc == '\r' and c == '\n': crlfs += 1 lc = c return rv def connect_tcp(hostport): rv = socket.socket() rv.connect(hostport) return rv @contextmanager def tmpfile(content): with NamedTemporaryFile('w', delete=False) as f: file_name = f.name f.write(str(content)) try: yield file_name finally: os.unlink(file_name) xattr_data = {} def _get_inode(fd): if not isinstance(fd, int): try: fd = fd.fileno() except AttributeError: return os.stat(fd).st_ino return os.fstat(fd).st_ino def _setxattr(fd, k, v): inode = _get_inode(fd) data = xattr_data.get(inode, {}) data[k] = v xattr_data[inode] = data def _getxattr(fd, k): inode = _get_inode(fd) data = xattr_data.get(inode, {}).get(k) if not data: raise IOError(errno.ENODATA, "Fake IOError") return data import xattr xattr.setxattr = _setxattr xattr.getxattr = _getxattr @contextmanager def temptree(files, contents=''): # generate enough contents to fill the files c = len(files) contents = (list(contents) + [''] * c)[:c] tempdir = mkdtemp() for path, content in zip(files, contents): if os.path.isabs(path): path = '.' + path new_path = os.path.join(tempdir, path) subdir = os.path.dirname(new_path) if not os.path.exists(subdir): os.makedirs(subdir) with open(new_path, 'w') as f: f.write(str(content)) try: yield tempdir finally: rmtree(tempdir) class NullLoggingHandler(logging.Handler): def emit(self, record): pass class UnmockTimeModule(object): """ Even if a test mocks time.time - you can restore unmolested behavior in a another module who imports time directly by monkey patching it's imported reference to the module with an instance of this class """ _orig_time = time.time def __getattribute__(self, name): if name == 'time': return UnmockTimeModule._orig_time return getattr(time, name) # logging.LogRecord.__init__ calls time.time logging.time = UnmockTimeModule() class FakeLogger(logging.Logger): # a thread safe logger def __init__(self, *args, **kwargs): self._clear() self.name = 'swift.unit.fake_logger' self.level = logging.NOTSET if 'facility' in kwargs: self.facility = kwargs['facility'] self.statsd_client = None self.thread_locals = None def _clear(self): self.log_dict = defaultdict(list) self.lines_dict = defaultdict(list) def _store_in(store_name): def stub_fn(self, *args, **kwargs): self.log_dict[store_name].append((args, kwargs)) return stub_fn def _store_and_log_in(store_name): def stub_fn(self, *args, **kwargs): self.log_dict[store_name].append((args, kwargs)) self._log(store_name, args[0], args[1:], **kwargs) return stub_fn def get_lines_for_level(self, level): return self.lines_dict[level] error = _store_and_log_in('error') info = _store_and_log_in('info') warning = _store_and_log_in('warning') warn = _store_and_log_in('warning') debug = _store_and_log_in('debug') def exception(self, *args, **kwargs): self.log_dict['exception'].append((args, kwargs, str(sys.exc_info()[1]))) print 'FakeLogger Exception: %s' % self.log_dict # mock out the StatsD logging methods: increment = _store_in('increment') decrement = _store_in('decrement') timing = _store_in('timing') timing_since = _store_in('timing_since') update_stats = _store_in('update_stats') set_statsd_prefix = _store_in('set_statsd_prefix') def get_increments(self): return [call[0][0] for call in self.log_dict['increment']] def get_increment_counts(self): counts = {} for metric in self.get_increments(): if metric not in counts: counts[metric] = 0 counts[metric] += 1 return counts def setFormatter(self, obj): self.formatter = obj def close(self): self._clear() def set_name(self, name): # don't touch _handlers self._name = name def acquire(self): pass def release(self): pass def createLock(self): pass def emit(self, record): pass def _handle(self, record): try: line = record.getMessage() except TypeError: print 'WARNING: unable to format log message %r %% %r' % ( record.msg, record.args) raise self.lines_dict[record.levelno].append(line) def handle(self, record): self._handle(record) def flush(self): pass def handleError(self, record): pass class DebugLogger(FakeLogger): """A simple stdout logging version of FakeLogger""" def __init__(self, *args, **kwargs): FakeLogger.__init__(self, *args, **kwargs) self.formatter = logging.Formatter("%(server)s: %(message)s") def handle(self, record): self._handle(record) print self.formatter.format(record) def debug_logger(name='test'): """get a named adapted debug logger""" return LogAdapter(DebugLogger(), name) original_syslog_handler = logging.handlers.SysLogHandler def fake_syslog_handler(): for attr in dir(original_syslog_handler): if attr.startswith('LOG'): setattr(FakeLogger, attr, copy.copy(getattr(logging.handlers.SysLogHandler, attr))) FakeLogger.priority_map = \ copy.deepcopy(logging.handlers.SysLogHandler.priority_map) logging.handlers.SysLogHandler = FakeLogger if config_true_value(get_config('unit_test').get('fake_syslog', 'False')): fake_syslog_handler() class MockTrue(object): """ Instances of MockTrue evaluate like True Any attr accessed on an instance of MockTrue will return a MockTrue instance. Any method called on an instance of MockTrue will return a MockTrue instance. >>> thing = MockTrue() >>> thing True >>> thing == True # True == True True >>> thing == False # True == False False >>> thing != True # True != True False >>> thing != False # True != False True >>> thing.attribute True >>> thing.method() True >>> thing.attribute.method() True >>> thing.method().attribute True """ def __getattribute__(self, *args, **kwargs): return self def __call__(self, *args, **kwargs): return self def __repr__(*args, **kwargs): return repr(True) def __eq__(self, other): return other is True def __ne__(self, other): return other is not True @contextmanager def mock(update): returns = [] deletes = [] for key, value in update.items(): imports = key.split('.') attr = imports.pop(-1) module = __import__(imports[0], fromlist=imports[1:]) for modname in imports[1:]: module = getattr(module, modname) if hasattr(module, attr): returns.append((module, attr, getattr(module, attr))) else: deletes.append((module, attr)) setattr(module, attr, value) try: yield True finally: for module, attr, value in returns: setattr(module, attr, value) for module, attr in deletes: delattr(module, attr) def fake_http_connect(*code_iter, **kwargs): class FakeConn(object): def __init__(self, status, etag=None, body='', timestamp='1', expect_status=None, headers=None): self.status = status if expect_status is None: self.expect_status = self.status else: self.expect_status = expect_status self.reason = 'Fake' self.host = '1.2.3.4' self.port = '1234' self.sent = 0 self.received = 0 self.etag = etag self.body = body self.headers = headers or {} self.timestamp = timestamp if 'slow' in kwargs and isinstance(kwargs['slow'], list): try: self._next_sleep = kwargs['slow'].pop(0) except IndexError: self._next_sleep = None def getresponse(self): if kwargs.get('raise_exc'): raise Exception('test') if kwargs.get('raise_timeout_exc'): raise Timeout() return self def getexpect(self): if self.expect_status == -2: raise HTTPException() if self.expect_status == -3: return FakeConn(507) if self.expect_status == -4: return FakeConn(201) if self.expect_status == 412: return FakeConn(412) return FakeConn(100) def getheaders(self): etag = self.etag if not etag: if isinstance(self.body, str): etag = '"' + md5(self.body).hexdigest() + '"' else: etag = '"68b329da9893e34099c7d8ad5cb9c940"' headers = {'content-length': len(self.body), 'content-type': 'x-application/test', 'x-timestamp': self.timestamp, 'last-modified': self.timestamp, 'x-object-meta-test': 'testing', 'x-delete-at': '9876543210', 'etag': etag, 'x-works': 'yes'} if self.status // 100 == 2: headers['x-account-container-count'] = \ kwargs.get('count', 12345) if not self.timestamp: del headers['x-timestamp'] try: if container_ts_iter.next() is False: headers['x-container-timestamp'] = '1' except StopIteration: pass am_slow, value = self.get_slow() if am_slow: headers['content-length'] = '4' headers.update(self.headers) return headers.items() def get_slow(self): if 'slow' in kwargs and isinstance(kwargs['slow'], list): if self._next_sleep is not None: return True, self._next_sleep else: return False, 0.01 if kwargs.get('slow') and isinstance(kwargs['slow'], Number): return True, kwargs['slow'] return bool(kwargs.get('slow')), 0.1 def read(self, amt=None): am_slow, value = self.get_slow() if am_slow: if self.sent < 4: self.sent += 1 sleep(value) return ' ' rv = self.body[:amt] self.body = self.body[amt:] return rv def send(self, amt=None): am_slow, value = self.get_slow() if am_slow: if self.received < 4: self.received += 1 sleep(value) def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) if isinstance(kwargs.get('headers'), list): headers_iter = iter(kwargs['headers']) else: headers_iter = iter([kwargs.get('headers', {})] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): x = [x] * len(code_iter) container_ts_iter = iter(x) code_iter = iter(code_iter) static_body = kwargs.get('body', None) body_iter = kwargs.get('body_iter', None) if body_iter: body_iter = iter(body_iter) def connect(*args, **ckwargs): if kwargs.get('slow_connect', False): sleep(0.1) if 'give_content_type' in kwargs: if len(args) >= 7 and 'Content-Type' in args[6]: kwargs['give_content_type'](args[6]['Content-Type']) else: kwargs['give_content_type']('') if 'give_connect' in kwargs: kwargs['give_connect'](*args, **ckwargs) status = code_iter.next() if isinstance(status, tuple): status, expect_status = status else: expect_status = status etag = etag_iter.next() headers = headers_iter.next() timestamp = timestamps_iter.next() if status <= 0: raise HTTPException() if body_iter is None: body = static_body or '' else: body = body_iter.next() return FakeConn(status, etag, body=body, timestamp=timestamp, expect_status=expect_status, headers=headers) connect.code_iter = code_iter return connect swift-1.13.1/test/sample.conf0000664000175400017540000000257112323703611017201 0ustar jenkinsjenkins00000000000000[func_test] # sample config auth_host = 127.0.0.1 auth_port = 8080 auth_ssl = no auth_prefix = /auth/ ## sample config for Swift with Keystone #auth_version = 2 #auth_host = localhost #auth_port = 5000 #auth_ssl = no #auth_prefix = /v2.0/ # Primary functional test account (needs admin access to the account) account = test username = tester password = testing # User on a second account (needs admin access to the account) account2 = test2 username2 = tester2 password2 = testing2 # User on same account as first, but without admin access username3 = tester3 password3 = testing3 # If not defined here, the test runner will try to use the default constraint # values as constructed by the constraints module, which will attempt to get # them from /etc/swift/swift.conf, if possible. Then, if the swift.conf file # isn't found, the test runner will skip tests that depend on those values. # Note that the cluster must have "sane" values for the test suite to pass. #max_file_size = 5368709122 #max_meta_name_length = 128 #max_meta_value_length = 256 #max_meta_count = 90 #max_meta_overall_size = 4096 #max_header_size = 8192 #max_object_name_length = 1024 #container_listing_limit = 10000 #account_listing_limit = 10000 #max_account_name_length = 256 #max_container_name_length = 256 collate = C [unit_test] fake_syslog = False [probe_test] # check_server_timeout = 30 # validate_rsync = false swift-1.13.1/test/functional/0000775000175400017540000000000012323703665017217 5ustar jenkinsjenkins00000000000000swift-1.13.1/test/functional/test_container.py0000775000175400017540000014125412323703614022616 0ustar jenkinsjenkins00000000000000#!/usr/bin/python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import json import unittest from nose import SkipTest from uuid import uuid4 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift_testing import check_response, retry, skip, skip2, skip3, \ swift_test_perm, web_front_end, requires_acls, swift_test_user class TestContainer(unittest.TestCase): def setUp(self): if skip: raise SkipTest self.name = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) def tearDown(self): if skip: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path + '/' + self.name + '?format=json', '', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, obj): conn.request('DELETE', '/'.join([parsed.path, self.name, obj['name']]), '', {'X-Auth-Token': token}) return check_response(conn) while True: resp = retry(get) body = resp.read() self.assert_(resp.status // 100 == 2, resp.status) objs = json.loads(body) if not objs: break for obj in objs: resp = retry(delete, obj) resp.read() self.assertEqual(resp.status, 204) def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) def test_multi_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, name: value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(post, 'X-Container-Meta-One', '1') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-one'), '1') resp = retry(post, 'X-Container-Meta-Two', '2') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-one'), '1') self.assertEqual(resp.getheader('x-container-meta-two'), '2') def test_unicode_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, name: value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) uni_key = u'X-Container-Meta-uni\u0E12' uni_value = u'uni\u0E12' if (web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Container-Meta-uni', uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('X-Container-Meta-uni'), uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), uni_value.encode('utf-8')) def test_PUT_metadata(self): if skip: raise SkipTest def put(url, token, parsed, conn, name, value): conn.request('PUT', parsed.path + '/' + name, '', {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) return check_response(conn) def head(url, token, parsed, conn, name): conn.request('HEAD', parsed.path + '/' + name, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/' + name, '', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', parsed.path + '/' + name, '', {'X-Auth-Token': token}) return check_response(conn) name = uuid4().hex resp = retry(put, name, 'Value') resp.read() self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry(put, name, '') resp.read() self.assertEqual(resp.status, 201) resp = retry(head, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get, name) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) def test_POST_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, value): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), None) resp = retry(post, 'Value') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') def test_PUT_bad_metadata(self): if skip: raise SkipTest def put(url, token, parsed, conn, name, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) conn.request('PUT', parsed.path + '/' + name, '', headers) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', parsed.path + '/' + name, '', {'X-Auth-Token': token}) return check_response(conn) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 404) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) name = uuid4().hex resp = retry( put, name, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} for x in xrange(MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) name = uuid4().hex headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 404) name = uuid4().hex headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Container-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 204) name = uuid4().hex headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(put, name, headers) resp.read() self.assertEqual(resp.status, 400) resp = retry(delete, name) resp.read() self.assertEqual(resp.status, 404) def test_POST_bad_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) conn.request('POST', parsed.path + '/' + self.name, '', headers) return check_response(conn) resp = retry( post, {'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, {'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() self.assertEqual(resp.status, 400) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Container-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Container-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers['X-Container-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) def test_public_container(self): if skip: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path + '/' + self.name) return check_response(conn) try: resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: self.assert_(str(err).startswith('No result after '), err) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': '.r:*,.rlistings'}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) resp = retry(get) resp.read() self.assertEqual(resp.status, 204) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': ''}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: self.assert_(str(err).startswith('No result after '), err) def test_cross_account_container(self): if skip or skip2: raise SkipTest # Obtain the first account's string first_account = ['unknown'] def get1(url, token, parsed, conn): first_account[0] = parsed.path conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get1) resp.read() # Ensure we can't access the container with the second account def get2(url, token, parsed, conn): conn.request('GET', first_account[0] + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 403) # Make the container accessible by the second account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[1], 'X-Container-Write': swift_test_perm[1]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can now use the container with the second account resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 204) # Make the container private again def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': '', 'X-Container-Write': ''}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can't access the container with the second account again resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 403) def test_cross_account_public_container(self): if skip or skip2: raise SkipTest # Obtain the first account's string first_account = ['unknown'] def get1(url, token, parsed, conn): first_account[0] = parsed.path conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get1) resp.read() # Ensure we can't access the container with the second account def get2(url, token, parsed, conn): conn.request('GET', first_account[0] + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 403) # Make the container completely public def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': '.r:*,.rlistings'}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can now read the container with the second account resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the second account def put2(url, token, parsed, conn): conn.request('PUT', first_account[0] + '/' + self.name + '/object', 'test object', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put2, use_account=2) resp.read() self.assertEqual(resp.status, 403) # Now make the container also writeable by the second account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Write': swift_test_perm[1]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can still read the container with the second account resp = retry(get2, use_account=2) resp.read() self.assertEqual(resp.status, 204) # And that we can now write with the second account resp = retry(put2, use_account=2) resp.read() self.assertEqual(resp.status, 201) def test_nonadmin_user(self): if skip or skip3: raise SkipTest # Obtain the first account's string first_account = ['unknown'] def get1(url, token, parsed, conn): first_account[0] = parsed.path conn.request('HEAD', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get1) resp.read() # Ensure we can't access the container with the third account def get3(url, token, parsed, conn): conn.request('GET', first_account[0] + '/' + self.name, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get3, use_account=3) resp.read() self.assertEqual(resp.status, 403) # Make the container accessible by the third account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can now read the container with the third account resp = retry(get3, use_account=3) resp.read() self.assertEqual(resp.status, 204) # But we shouldn't be able to write with the third account def put3(url, token, parsed, conn): conn.request('PUT', first_account[0] + '/' + self.name + '/object', 'test object', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put3, use_account=3) resp.read() self.assertEqual(resp.status, 403) # Now make the container also writeable by the third account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Write': swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # Ensure we can still read the container with the third account resp = retry(get3, use_account=3) resp.read() self.assertEqual(resp.status, 204) # And that we can now write with the third account resp = retry(put3, use_account=3) resp.read() self.assertEqual(resp.status, 201) @requires_acls def test_read_only_acl_listings(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) # cannot list containers resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-only access acl_user = swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-only can list containers resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.name in listing) # read-only can not create containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 403) # but it can see newly created ones resp = retry(put, new_container_name, use_account=1) resp.read() self.assertEquals(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(new_container_name in listing) @requires_acls def test_read_only_acl_metadata(self): if skip3: raise SkipTest def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def post(url, token, parsed, conn, name, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path + '/%s' % name, '', new_headers) return check_response(conn) # add some metadata value = str(uuid4()) headers = {'x-container-meta-test': value} resp = retry(post, self.name, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-only access acl_user = swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-only can NOT write container metadata new_value = str(uuid4()) headers = {'x-container-meta-test': new_value} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 403) # read-only can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) @requires_acls def test_read_write_acl_listings(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) # cannot list containers resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-write access acl_user = swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can list containers resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.name in listing) # can create new containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(new_container_name in listing) # can also delete them resp = retry(delete, new_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(new_container_name not in listing) # even if they didn't create them empty_container_name = str(uuid4()) resp = retry(put, empty_container_name, use_account=1) resp.read() self.assertEquals(resp.status, 201) resp = retry(delete, empty_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 204) @requires_acls def test_read_write_acl_metadata(self): if skip3: raise SkipTest def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def post(url, token, parsed, conn, name, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path + '/%s' % name, '', new_headers) return check_response(conn) # add some metadata value = str(uuid4()) headers = {'x-container-meta-test': value} resp = retry(post, self.name, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-write access acl_user = swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-write can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # read-write can also write container metadata new_value = str(uuid4()) headers = {'x-container-meta-test': new_value} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and remove it headers = {'x-remove-container-meta-test': 'true'} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) @requires_acls def test_admin_acl_listing(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) # cannot list containers resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant admin access acl_user = swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can list containers resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.name in listing) # can create new containers new_container_name = str(uuid4()) resp = retry(put, new_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 201) resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(new_container_name in listing) # can also delete them resp = retry(delete, new_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(new_container_name not in listing) # even if they didn't create them empty_container_name = str(uuid4()) resp = retry(put, empty_container_name, use_account=1) resp.read() self.assertEquals(resp.status, 201) resp = retry(delete, empty_container_name, use_account=3) resp.read() self.assertEquals(resp.status, 204) @requires_acls def test_admin_acl_metadata(self): if skip3: raise SkipTest def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def post(url, token, parsed, conn, name, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path + '/%s' % name, '', new_headers) return check_response(conn) # add some metadata value = str(uuid4()) headers = {'x-container-meta-test': value} resp = retry(post, self.name, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # cannot see metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant access acl_user = swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # can also write container metadata new_value = str(uuid4()) headers = {'x-container-meta-test': new_value} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and remove it headers = {'x-remove-container-meta-test': 'true'} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEquals(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) @requires_acls def test_protected_container_sync(self): if skip3: raise SkipTest def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def post(url, token, parsed, conn, name, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path + '/%s' % name, '', new_headers) return check_response(conn) # add some metadata value = str(uuid4()) headers = { 'x-container-sync-key': 'secret', 'x-container-meta-test': value, } resp = retry(post, self.name, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # grant read-only access acl_user = swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) # and can not write headers = {'x-container-sync-key': str(uuid4())} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 403) # grant read-write access acl_user = swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) # sanity check sync-key w/ account1 resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') # and can write new_value = str(uuid4()) headers = { 'x-container-sync-key': str(uuid4()), 'x-container-meta-test': new_value, } resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) # validate w/ account1 resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # but can not write sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') # grant admin access acl_user = swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # admin can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and ALSO sync-key self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') # admin tester3 can even change sync-key new_secret = str(uuid4()) headers = {'x-container-sync-key': new_secret} resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret) @requires_acls def test_protected_container_acl(self): if skip3: raise SkipTest def get(url, token, parsed, conn, name): conn.request('GET', parsed.path + '/%s' % name, '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def post(url, token, parsed, conn, name, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path + '/%s' % name, '', new_headers) return check_response(conn) # add some container acls value = str(uuid4()) headers = { 'x-container-read': 'jdoe', 'x-container-write': 'jdoe', 'x-container-meta-test': value, } resp = retry(post, self.name, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # grant read-only access acl_user = swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not container acl self.assertEqual(resp.getheader('X-Container-Read'), None) self.assertEqual(resp.getheader('X-Container-Write'), None) # and can not write headers = { 'x-container-read': 'frank', 'x-container-write': 'frank', } resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 403) # grant read-write access acl_user = swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) # but not container acl self.assertEqual(resp.getheader('X-Container-Read'), None) self.assertEqual(resp.getheader('X-Container-Write'), None) # sanity check container acls with account1 resp = retry(get, self.name, use_account=1) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') # and can write new_value = str(uuid4()) headers = { 'x-container-read': 'frank', 'x-container-write': 'frank', 'x-container-meta-test': new_value, } resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=1) # validate w/ account1 resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # but can not write container acls self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') # grant admin access acl_user = swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # admin can read container metadata resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) # and ALSO container acls self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') # admin tester3 can even change container acls new_value = str(uuid4()) headers = { 'x-container-read': '.r:*', } resp = retry(post, self.name, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, self.name, use_account=3) resp.read() self.assertEquals(resp.status, 204) self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') def test_long_name_content_type(self): if skip: raise SkipTest def put(url, token, parsed, conn): container_name = 'X' * 2048 conn.request('PUT', '%s/%s' % (parsed.path, container_name), 'there', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 400) self.assertEqual(resp.getheader('Content-Type'), 'text/html; charset=UTF-8') def test_null_name(self): if skip: raise SkipTest def put(url, token, parsed, conn): conn.request('PUT', '%s/abc%%00def' % parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) if (web_front_end == 'apache2'): self.assertEqual(resp.status, 404) else: self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') self.assertEqual(resp.status, 412) if __name__ == '__main__': unittest.main() swift-1.13.1/test/functional/swift_test_client.py0000664000175400017540000006635212323703614023330 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import hashlib import httplib import os import random import socket import StringIO import time import urllib import simplejson as json from nose import SkipTest from xml.dom import minidom from swiftclient import get_auth from test import safe_repr class AuthenticationFailed(Exception): pass class RequestError(Exception): pass class ResponseError(Exception): def __init__(self, response, method=None, path=None): self.status = response.status self.reason = response.reason self.method = method self.path = path self.headers = response.getheaders() for name, value in self.headers: if name.lower() == 'x-trans-id': self.txid = value break else: self.txid = None super(ResponseError, self).__init__() def __str__(self): return repr(self) def __repr__(self): return '%d: %r (%r %r) txid=%s' % ( self.status, self.reason, self.method, self.path, self.txid) def listing_empty(method): for i in xrange(6): if len(method()) == 0: return True time.sleep(2 ** i) return False def listing_items(method): marker = None once = True items = [] while once or items: for i in items: yield i if once or marker: if marker: items = method(parms={'marker': marker}) else: items = method() if len(items) == 10000: marker = items[-1] else: marker = None once = False else: items = [] class Connection(object): def __init__(self, config): for key in 'auth_host auth_port auth_ssl username password'.split(): if key not in config: raise SkipTest self.auth_host = config['auth_host'] self.auth_port = int(config['auth_port']) self.auth_ssl = config['auth_ssl'] in ('on', 'true', 'yes', '1') self.auth_prefix = config.get('auth_prefix', '/') self.auth_version = str(config.get('auth_version', '1')) self.account = config.get('account') self.username = config['username'] self.password = config['password'] self.storage_host = None self.storage_port = None self.conn_class = None def get_account(self): return Account(self, self.account) def authenticate(self, clone_conn=None): if clone_conn: self.conn_class = clone_conn.conn_class self.storage_host = clone_conn.storage_host self.storage_url = clone_conn.storage_url self.storage_port = clone_conn.storage_port self.storage_token = clone_conn.storage_token return if self.auth_version == "1": auth_path = '%sv1.0' % (self.auth_prefix) if self.account: auth_user = '%s:%s' % (self.account, self.username) else: auth_user = self.username else: auth_user = self.username auth_path = self.auth_prefix auth_scheme = 'https://' if self.auth_ssl else 'http://' auth_netloc = "%s:%d" % (self.auth_host, self.auth_port) auth_url = auth_scheme + auth_netloc + auth_path (storage_url, storage_token) = get_auth( auth_url, auth_user, self.password, snet=False, tenant_name=self.account, auth_version=self.auth_version, os_options={}) if not (storage_url and storage_token): raise AuthenticationFailed() x = storage_url.split('/') if x[0] == 'http:': self.conn_class = httplib.HTTPConnection self.storage_port = 80 elif x[0] == 'https:': self.conn_class = httplib.HTTPSConnection self.storage_port = 443 else: raise ValueError('unexpected protocol %s' % (x[0])) self.storage_host = x[2].split(':')[0] if ':' in x[2]: self.storage_port = int(x[2].split(':')[1]) # Make sure storage_url is a string and not unicode, since # keystoneclient (called by swiftclient) returns them in # unicode and this would cause troubles when doing # no_safe_quote query. self.storage_url = str('/%s/%s' % (x[3], x[4])) self.storage_token = storage_token self.http_connect() return self.storage_url, self.storage_token def cluster_info(self): """ Retrieve the data in /info, or {} on 404 """ status = self.make_request('GET', '/info', cfg={'absolute_path': True}) if status == 404: return {} if not 200 <= status <= 299: raise ResponseError(self.response, 'GET', '/info') return json.loads(self.response.read()) def http_connect(self): self.connection = self.conn_class(self.storage_host, port=self.storage_port) #self.connection.set_debuglevel(3) def make_path(self, path=[], cfg={}): if cfg.get('version_only_path'): return '/' + self.storage_url.split('/')[1] if path: quote = urllib.quote if cfg.get('no_quote') or cfg.get('no_path_quote'): quote = lambda x: x return '%s/%s' % (self.storage_url, '/'.join([quote(i) for i in path])) else: return self.storage_url def make_headers(self, hdrs, cfg={}): headers = {} if not cfg.get('no_auth_token'): headers['X-Auth-Token'] = self.storage_token if isinstance(hdrs, dict): headers.update(hdrs) return headers def make_request(self, method, path=[], data='', hdrs={}, parms={}, cfg={}): if not cfg.get('absolute_path'): # Set absolute_path=True to make a request to exactly the given # path, not storage path + given path. Useful for # non-account/container/object requests. path = self.make_path(path, cfg=cfg) headers = self.make_headers(hdrs, cfg=cfg) if isinstance(parms, dict) and parms: quote = urllib.quote if cfg.get('no_quote') or cfg.get('no_parms_quote'): quote = lambda x: x query_args = ['%s=%s' % (quote(x), quote(str(y))) for (x, y) in parms.items()] path = '%s?%s' % (path, '&'.join(query_args)) if not cfg.get('no_content_length'): if cfg.get('set_content_length'): headers['Content-Length'] = cfg.get('set_content_length') else: headers['Content-Length'] = len(data) def try_request(): self.http_connect() self.connection.request(method, path, data, headers) return self.connection.getresponse() self.response = None try_count = 0 fail_messages = [] while try_count < 5: try_count += 1 try: self.response = try_request() except httplib.HTTPException as e: fail_messages.append(safe_repr(e)) continue if self.response.status == 401: fail_messages.append("Response 401") self.authenticate() continue elif self.response.status == 503: fail_messages.append("Response 503") if try_count != 5: time.sleep(5) continue break if self.response: return self.response.status request = "{method} {path} headers: {headers} data: {data}".format( method=method, path=path, headers=headers, data=data) raise RequestError('Unable to complete http request: %s. ' 'Attempts: %s, Failures: %s' % (request, len(fail_messages), fail_messages)) def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False): self.http_connect() path = self.make_path(path, cfg) headers = self.make_headers(hdrs, cfg=cfg) if chunked: headers['Transfer-Encoding'] = 'chunked' headers.pop('Content-Length', None) if isinstance(parms, dict) and parms: quote = urllib.quote if cfg.get('no_quote') or cfg.get('no_parms_quote'): quote = lambda x: x query_args = ['%s=%s' % (quote(x), quote(str(y))) for (x, y) in parms.items()] path = '%s?%s' % (path, '&'.join(query_args)) self.connection = self.conn_class(self.storage_host, port=self.storage_port) #self.connection.set_debuglevel(3) self.connection.putrequest('PUT', path) for key, value in headers.iteritems(): self.connection.putheader(key, value) self.connection.endheaders() def put_data(self, data, chunked=False): if chunked: self.connection.send('%x\r\n%s\r\n' % (len(data), data)) else: self.connection.send(data) def put_end(self, chunked=False): if chunked: self.connection.send('0\r\n\r\n') self.response = self.connection.getresponse() self.connection.close() return self.response.status class Base(object): def __str__(self): return self.name def header_fields(self, required_fields, optional_fields=()): headers = dict(self.conn.response.getheaders()) ret = {} for field in required_fields: if field[1] not in headers: raise ValueError("%s was not found in response header" % (field[1])) try: ret[field[0]] = int(headers[field[1]]) except ValueError: ret[field[0]] = headers[field[1]] for field in optional_fields: if field[1] not in headers: continue try: ret[field[0]] = int(headers[field[1]]) except ValueError: ret[field[0]] = headers[field[1]] return ret class Account(Base): def __init__(self, conn, name): self.conn = conn self.name = str(name) def update_metadata(self, metadata={}, cfg={}): headers = dict(("X-Account-Meta-%s" % k, v) for k, v in metadata.items()) self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) if not 200 <= self.conn.response.status <= 299: raise ResponseError(self.conn.response, 'POST', self.conn.make_path(self.path)) return True def container(self, container_name): return Container(self.conn, self.name, container_name) def containers(self, hdrs={}, parms={}, cfg={}): format_type = parms.get('format', None) if format_type not in [None, 'json', 'xml']: raise RequestError('Invalid format: %s' % format_type) if format_type is None and 'format' in parms: del parms['format'] status = self.conn.make_request('GET', self.path, hdrs=hdrs, parms=parms, cfg=cfg) if status == 200: if format_type == 'json': conts = json.loads(self.conn.response.read()) for cont in conts: cont['name'] = cont['name'].encode('utf-8') return conts elif format_type == 'xml': conts = [] tree = minidom.parseString(self.conn.response.read()) for x in tree.getElementsByTagName('container'): cont = {} for key in ['name', 'count', 'bytes']: cont[key] = x.getElementsByTagName(key)[0].\ childNodes[0].nodeValue conts.append(cont) for cont in conts: cont['name'] = cont['name'].encode('utf-8') return conts else: lines = self.conn.response.read().split('\n') if lines and not lines[-1]: lines = lines[:-1] return lines elif status == 204: return [] raise ResponseError(self.conn.response, 'GET', self.conn.make_path(self.path)) def delete_containers(self): for c in listing_items(self.containers): cont = self.container(c) if not cont.delete_recursive(): return False return listing_empty(self.containers) def info(self, hdrs={}, parms={}, cfg={}): if self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) != 204: raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) fields = [['object_count', 'x-account-object-count'], ['container_count', 'x-account-container-count'], ['bytes_used', 'x-account-bytes-used']] return self.header_fields(fields) @property def path(self): return [] class Container(Base): def __init__(self, conn, account, name): self.conn = conn self.account = str(account) self.name = str(name) def create(self, hdrs={}, parms={}, cfg={}): return self.conn.make_request('PUT', self.path, hdrs=hdrs, parms=parms, cfg=cfg) in (201, 202) def delete(self, hdrs={}, parms={}): return self.conn.make_request('DELETE', self.path, hdrs=hdrs, parms=parms) == 204 def delete_files(self): for f in listing_items(self.files): file_item = self.file(f) if not file_item.delete(): return False return listing_empty(self.files) def delete_recursive(self): return self.delete_files() and self.delete() def file(self, file_name): return File(self.conn, self.account, self.name, file_name) def files(self, hdrs={}, parms={}, cfg={}): format_type = parms.get('format', None) if format_type not in [None, 'json', 'xml']: raise RequestError('Invalid format: %s' % format_type) if format_type is None and 'format' in parms: del parms['format'] status = self.conn.make_request('GET', self.path, hdrs=hdrs, parms=parms, cfg=cfg) if status == 200: if format_type == 'json': files = json.loads(self.conn.response.read()) for file_item in files: file_item['name'] = file_item['name'].encode('utf-8') file_item['content_type'] = file_item['content_type'].\ encode('utf-8') return files elif format_type == 'xml': files = [] tree = minidom.parseString(self.conn.response.read()) for x in tree.getElementsByTagName('object'): file_item = {} for key in ['name', 'hash', 'bytes', 'content_type', 'last_modified']: file_item[key] = x.getElementsByTagName(key)[0].\ childNodes[0].nodeValue files.append(file_item) for file_item in files: file_item['name'] = file_item['name'].encode('utf-8') file_item['content_type'] = file_item['content_type'].\ encode('utf-8') return files else: content = self.conn.response.read() if content: lines = content.split('\n') if lines and not lines[-1]: lines = lines[:-1] return lines else: return [] elif status == 204: return [] raise ResponseError(self.conn.response, 'GET', self.conn.make_path(self.path)) def info(self, hdrs={}, parms={}, cfg={}): self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) if self.conn.response.status == 204: required_fields = [['bytes_used', 'x-container-bytes-used'], ['object_count', 'x-container-object-count']] optional_fields = [['versions', 'x-versions-location']] return self.header_fields(required_fields, optional_fields) raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) @property def path(self): return [self.name] class File(Base): def __init__(self, conn, account, container, name): self.conn = conn self.account = str(account) self.container = str(container) self.name = str(name) self.chunked_write_in_progress = False self.content_type = None self.size = None self.metadata = {} def make_headers(self, cfg={}): headers = {} if not cfg.get('no_content_length'): if cfg.get('set_content_length'): headers['Content-Length'] = cfg.get('set_content_length') elif self.size: headers['Content-Length'] = self.size else: headers['Content-Length'] = 0 if cfg.get('no_content_type'): pass elif self.content_type: headers['Content-Type'] = self.content_type else: headers['Content-Type'] = 'application/octet-stream' for key in self.metadata: headers['X-Object-Meta-' + key] = self.metadata[key] return headers @classmethod def compute_md5sum(cls, data): block_size = 4096 if isinstance(data, str): data = StringIO.StringIO(data) checksum = hashlib.md5() buff = data.read(block_size) while buff: checksum.update(buff) buff = data.read(block_size) data.seek(0) return checksum.hexdigest() def copy(self, dest_cont, dest_file, hdrs={}, parms={}, cfg={}): if 'destination' in cfg: headers = {'Destination': cfg['destination']} elif cfg.get('no_destination'): headers = {} else: headers = {'Destination': '%s/%s' % (dest_cont, dest_file)} headers.update(hdrs) if 'Destination' in headers: headers['Destination'] = urllib.quote(headers['Destination']) return self.conn.make_request('COPY', self.path, hdrs=headers, parms=parms) == 201 def delete(self, hdrs={}, parms={}): if self.conn.make_request('DELETE', self.path, hdrs=hdrs, parms=parms) != 204: raise ResponseError(self.conn.response, 'DELETE', self.conn.make_path(self.path)) return True def info(self, hdrs={}, parms={}, cfg={}): if self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms, cfg=cfg) != 200: raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) fields = [['content_length', 'content-length'], ['content_type', 'content-type'], ['last_modified', 'last-modified'], ['etag', 'etag']] header_fields = self.header_fields(fields) header_fields['etag'] = header_fields['etag'].strip('"') return header_fields def initialize(self, hdrs={}, parms={}): if not self.name: return False status = self.conn.make_request('HEAD', self.path, hdrs=hdrs, parms=parms) if status == 404: return False elif (status < 200) or (status > 299): raise ResponseError(self.conn.response, 'HEAD', self.conn.make_path(self.path)) for hdr in self.conn.response.getheaders(): if hdr[0].lower() == 'content-type': self.content_type = hdr[1] if hdr[0].lower().startswith('x-object-meta-'): self.metadata[hdr[0][14:]] = hdr[1] if hdr[0].lower() == 'etag': self.etag = hdr[1].strip('"') if hdr[0].lower() == 'content-length': self.size = int(hdr[1]) if hdr[0].lower() == 'last-modified': self.last_modified = hdr[1] return True def load_from_filename(self, filename, callback=None): fobj = open(filename, 'rb') self.write(fobj, callback=callback) fobj.close() @property def path(self): return [self.container, self.name] @classmethod def random_data(cls, size=None): if size is None: size = random.randint(1, 32768) fd = open('/dev/urandom', 'r') data = fd.read(size) fd.close() return data def read(self, size=-1, offset=0, hdrs=None, buffer=None, callback=None, cfg={}, parms={}): if size > 0: range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1) if hdrs: hdrs['Range'] = range_string else: hdrs = {'Range': range_string} status = self.conn.make_request('GET', self.path, hdrs=hdrs, cfg=cfg, parms=parms) if (status < 200) or (status > 299): raise ResponseError(self.conn.response, 'GET', self.conn.make_path(self.path)) for hdr in self.conn.response.getheaders(): if hdr[0].lower() == 'content-type': self.content_type = hdr[1] if hasattr(buffer, 'write'): scratch = self.conn.response.read(8192) transferred = 0 while len(scratch) > 0: buffer.write(scratch) transferred += len(scratch) if callable(callback): callback(transferred, self.size) scratch = self.conn.response.read(8192) return None else: return self.conn.response.read() def read_md5(self): status = self.conn.make_request('GET', self.path) if (status < 200) or (status > 299): raise ResponseError(self.conn.response, 'GET', self.conn.make_path(self.path)) checksum = hashlib.md5() scratch = self.conn.response.read(8192) while len(scratch) > 0: checksum.update(scratch) scratch = self.conn.response.read(8192) return checksum.hexdigest() def save_to_filename(self, filename, callback=None): try: fobj = open(filename, 'wb') self.read(buffer=fobj, callback=callback) finally: fobj.close() def sync_metadata(self, metadata={}, cfg={}): self.metadata.update(metadata) if self.metadata: headers = self.make_headers(cfg=cfg) if not cfg.get('no_content_length'): if cfg.get('set_content_length'): headers['Content-Length'] = \ cfg.get('set_content_length') else: headers['Content-Length'] = 0 self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) if self.conn.response.status not in (201, 202): raise ResponseError(self.conn.response, 'POST', self.conn.make_path(self.path)) return True def chunked_write(self, data=None, hdrs={}, parms={}, cfg={}): if data is not None and self.chunked_write_in_progress: self.conn.put_data(data, True) elif data is not None: self.chunked_write_in_progress = True headers = self.make_headers(cfg=cfg) headers.update(hdrs) self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg, chunked=True) self.conn.put_data(data, True) elif self.chunked_write_in_progress: self.chunked_write_in_progress = False return self.conn.put_end(True) == 201 else: raise RuntimeError def write(self, data='', hdrs={}, parms={}, callback=None, cfg={}, return_resp=False): block_size = 2 ** 20 if isinstance(data, file): try: data.flush() data.seek(0) except IOError: pass self.size = int(os.fstat(data.fileno())[6]) else: data = StringIO.StringIO(data) self.size = data.len headers = self.make_headers(cfg=cfg) headers.update(hdrs) self.conn.put_start(self.path, hdrs=headers, parms=parms, cfg=cfg) transferred = 0 buff = data.read(block_size) try: while len(buff) > 0: self.conn.put_data(buff) buff = data.read(block_size) transferred += len(buff) if callable(callback): callback(transferred, self.size) self.conn.put_end() except socket.timeout as err: raise err if (self.conn.response.status < 200) or \ (self.conn.response.status > 299): raise ResponseError(self.conn.response, 'PUT', self.conn.make_path(self.path)) try: data.seek(0) except IOError: pass self.md5 = self.compute_md5sum(data) if return_resp: return self.conn.response return True def write_random(self, size=None, hdrs={}, parms={}, cfg={}): data = self.random_data(size) if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): raise ResponseError(self.conn.response, 'PUT', self.conn.make_path(self.path)) self.md5 = self.compute_md5sum(StringIO.StringIO(data)) return data def write_random_return_resp(self, size=None, hdrs={}, parms={}, cfg={}): data = self.random_data(size) resp = self.write(data, hdrs=hdrs, parms=parms, cfg=cfg, return_resp=True) if not resp: raise ResponseError(self.conn.response) self.md5 = self.compute_md5sum(StringIO.StringIO(data)) return resp swift-1.13.1/test/functional/swift_testing.py0000664000175400017540000001717012323703614022462 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from httplib import HTTPException import os import socket import sys from time import sleep from urlparse import urlparse import functools from nose import SkipTest from test import get_config from swiftclient import get_auth, http_connection from test.functional.swift_test_client import Connection conf = get_config('func_test') web_front_end = conf.get('web_front_end', 'integral') normalized_urls = conf.get('normalized_urls', False) # If no conf was read, we will fall back to old school env vars swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] swift_test_tenant = ['', '', ''] swift_test_perm = ['', '', ''] if conf: swift_test_auth_version = str(conf.get('auth_version', '1')) swift_test_auth = 'http' if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): swift_test_auth = 'https' if 'auth_prefix' not in conf: conf['auth_prefix'] = '/' try: suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf swift_test_auth += suffix except KeyError: pass # skip if swift_test_auth_version == "1": swift_test_auth += 'v1.0' if 'account' in conf: swift_test_user[0] = '%(account)s:%(username)s' % conf else: swift_test_user[0] = '%(username)s' % conf swift_test_key[0] = conf['password'] try: swift_test_user[1] = '%s%s' % ( '%s:' % conf['account2'] if 'account2' in conf else '', conf['username2']) swift_test_key[1] = conf['password2'] except KeyError as err: pass # old conf, no second account tests can be run try: swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account' in conf else '', conf['username3']) swift_test_key[2] = conf['password3'] except KeyError as err: pass # old conf, no third account tests can be run for _ in range(3): swift_test_perm[_] = swift_test_user[_] else: swift_test_user[0] = conf['username'] swift_test_tenant[0] = conf['account'] swift_test_key[0] = conf['password'] swift_test_user[1] = conf['username2'] swift_test_tenant[1] = conf['account2'] swift_test_key[1] = conf['password2'] swift_test_user[2] = conf['username3'] swift_test_tenant[2] = conf['account'] swift_test_key[2] = conf['password3'] for _ in range(3): swift_test_perm[_] = swift_test_tenant[_] + ':' \ + swift_test_user[_] skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) if skip: print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) if not skip and skip2: print >>sys.stderr, \ 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) if not skip and skip3: print >>sys.stderr, \ 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' class AuthError(Exception): pass class InternalServerError(Exception): pass url = [None, None, None] token = [None, None, None] parsed = [None, None, None] conn = [None, None, None] def retry(func, *args, **kwargs): """ You can use the kwargs to override: 'retries' (default: 5) 'use_account' (default: 1) - which user's token to pass 'url_account' (default: matches 'use_account') - which user's storage URL 'resource' (default: url[url_account] - URL to connect to; retry() will interpolate the variable :storage_url: if present """ global url, token, parsed, conn retries = kwargs.get('retries', 5) attempts, backoff = 0, 1 # use account #1 by default; turn user's 1-indexed account into 0-indexed use_account = kwargs.pop('use_account', 1) - 1 # access our own account by default url_account = kwargs.pop('url_account', use_account + 1) - 1 while attempts <= retries: attempts += 1 try: if not url[use_account] or not token[use_account]: url[use_account], token[use_account] = \ get_auth(swift_test_auth, swift_test_user[use_account], swift_test_key[use_account], snet=False, tenant_name=swift_test_tenant[use_account], auth_version=swift_test_auth_version, os_options={}) parsed[use_account] = conn[use_account] = None if not parsed[use_account] or not conn[use_account]: parsed[use_account], conn[use_account] = \ http_connection(url[use_account]) # default resource is the account url[url_account] resource = kwargs.pop('resource', '%(storage_url)s') template_vars = {'storage_url': url[url_account]} parsed_result = urlparse(resource % template_vars) return func(url[url_account], token[use_account], parsed_result, conn[url_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: raise parsed[use_account] = conn[use_account] = None except AuthError: url[use_account] = token[use_account] = None continue except InternalServerError: pass if attempts <= retries: sleep(backoff) backoff *= 2 raise Exception('No result after %s retries.' % retries) def check_response(conn): resp = conn.getresponse() if resp.status == 401: resp.read() raise AuthError() elif resp.status // 100 == 5: resp.read() raise InternalServerError() return resp cluster_info = {} def get_cluster_info(): conn = Connection(conf) conn.authenticate() global cluster_info cluster_info = conn.cluster_info() def reset_acl(): def post(url, token, parsed, conn): conn.request('POST', parsed.path, '', { 'X-Auth-Token': token, 'X-Account-Access-Control': '{}' }) return check_response(conn) resp = retry(post, use_account=1) resp.read() def requires_acls(f): @functools.wraps(f) def wrapper(*args, **kwargs): if skip: raise SkipTest if not cluster_info: get_cluster_info() # Determine whether this cluster has account ACLs; if not, skip test if not cluster_info.get('tempauth', {}).get('account_acls'): raise SkipTest if 'keystoneauth' in cluster_info: # remove when keystoneauth supports account acls raise SkipTest reset_acl() try: rv = f(*args, **kwargs) finally: reset_acl() return rv return wrapper swift-1.13.1/test/functional/tests.py0000664000175400017540000026174312323703614020742 0ustar jenkinsjenkins00000000000000#!/usr/bin/python -u # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from datetime import datetime import hashlib import hmac import json import locale import random import StringIO import time import threading import unittest import urllib import uuid from nose import SkipTest from test import get_config from test.functional.swift_test_client import Account, Connection, File, \ ResponseError from swift.common import constraints config = get_config('func_test') for k in constraints.DEFAULT_CONSTRAINTS: if k in config: # prefer what's in test.conf config[k] = int(config[k]) elif constraints.SWIFT_CONSTRAINTS_LOADED: # swift.conf exists, so use what's defined there (or swift defaults) # This normally happens when the test is running locally to the cluster # as in a SAIO. config[k] = constraints.EFFECTIVE_CONSTRAINTS[k] else: # .functests don't know what the constraints of the tested cluster are, # so the tests can't reliably pass or fail. Therefore, skip those # tests. config[k] = '%s constraint is not defined' % k web_front_end = config.get('web_front_end', 'integral') normalized_urls = config.get('normalized_urls', False) def load_constraint(name): c = config[name] if not isinstance(c, int): raise SkipTest(c) return c locale.setlocale(locale.LC_COLLATE, config.get('collate', 'C')) def chunks(s, length=3): i, j = 0, length while i < len(s): yield s[i:j] i, j = j, j + length def timeout(seconds, method, *args, **kwargs): class TimeoutThread(threading.Thread): def __init__(self, method, *args, **kwargs): threading.Thread.__init__(self) self.method = method self.args = args self.kwargs = kwargs self.exception = None def run(self): try: self.method(*self.args, **self.kwargs) except Exception as e: self.exception = e t = TimeoutThread(method, *args, **kwargs) t.start() t.join(seconds) if t.exception: raise t.exception if t.isAlive(): t._Thread__stop() return True return False class Utils(object): @classmethod def create_ascii_name(cls, length=None): return uuid.uuid4().hex @classmethod def create_utf8_name(cls, length=None): if length is None: length = 15 else: length = int(length) utf8_chars = u'\uF10F\uD20D\uB30B\u9409\u8508\u5605\u3703\u1801'\ u'\u0900\uF110\uD20E\uB30C\u940A\u8509\u5606\u3704'\ u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ u'\u5608\u3706\u1804\u0903\u03A9\u2603' return ''.join([random.choice(utf8_chars) for x in xrange(length)]).encode('utf-8') create_name = create_ascii_name class Base(unittest.TestCase): def setUp(self): cls = type(self) if not cls.set_up: cls.env.setUp() cls.set_up = True def assert_body(self, body): response_body = self.env.conn.response.read() self.assert_(response_body == body, 'Body returned: %s' % (response_body)) def assert_status(self, status_or_statuses): self.assert_(self.env.conn.response.status == status_or_statuses or (hasattr(status_or_statuses, '__iter__') and self.env.conn.response.status in status_or_statuses), 'Status returned: %d Expected: %s' % (self.env.conn.response.status, status_or_statuses)) class Base2(object): def setUp(self): Utils.create_name = Utils.create_utf8_name super(Base2, self).setUp() def tearDown(self): Utils.create_name = Utils.create_ascii_name class TestAccountEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.containers = [] for i in range(10): cont = cls.account.container(Utils.create_name()) if not cont.create(): raise ResponseError(cls.conn.response) cls.containers.append(cont) class TestAccountDev(Base): env = TestAccountEnv set_up = False class TestAccountDevUTF8(Base2, TestAccountDev): set_up = False class TestAccount(Base): env = TestAccountEnv set_up = False def testNoAuthToken(self): self.assertRaises(ResponseError, self.env.account.info, cfg={'no_auth_token': True}) self.assert_status([401, 412]) self.assertRaises(ResponseError, self.env.account.containers, cfg={'no_auth_token': True}) self.assert_status([401, 412]) def testInvalidUTF8Path(self): invalid_utf8 = Utils.create_utf8_name()[::-1] container = self.env.account.container(invalid_utf8) self.assert_(not container.create(cfg={'no_path_quote': True})) self.assert_status(412) self.assert_body('Invalid UTF8 or contains NULL') def testVersionOnlyPath(self): self.env.account.conn.make_request('PUT', cfg={'version_only_path': True}) self.assert_status(412) self.assert_body('Bad URL') def testInvalidPath(self): was_url = self.env.account.conn.storage_url if (normalized_urls): self.env.account.conn.storage_url = '/' else: self.env.account.conn.storage_url = "/%s" % was_url self.env.account.conn.make_request('GET') try: self.assert_status(404) finally: self.env.account.conn.storage_url = was_url def testPUT(self): self.env.account.conn.make_request('PUT') self.assert_status([403, 405]) def testAccountHead(self): try_count = 0 while try_count < 5: try_count += 1 info = self.env.account.info() for field in ['object_count', 'container_count', 'bytes_used']: self.assert_(info[field] >= 0) if info['container_count'] == len(self.env.containers): break if try_count < 5: time.sleep(1) self.assertEqual(info['container_count'], len(self.env.containers)) self.assert_status(204) def testContainerSerializedInfo(self): container_info = {} for container in self.env.containers: info = {'bytes': 0} info['count'] = random.randint(10, 30) for i in range(info['count']): file_item = container.file(Utils.create_name()) bytes = random.randint(1, 32768) file_item.write_random(bytes) info['bytes'] += bytes container_info[container.name] = info for format_type in ['json', 'xml']: for a in self.env.account.containers( parms={'format': format_type}): self.assert_(a['count'] >= 0) self.assert_(a['bytes'] >= 0) headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': self.assertEqual(headers['content-type'], 'application/json; charset=utf-8') elif format_type == 'xml': self.assertEqual(headers['content-type'], 'application/xml; charset=utf-8') def testListingLimit(self): limit = load_constraint('account_listing_limit') for l in (1, 100, limit / 2, limit - 1, limit, limit + 1, limit * 2): p = {'limit': l} if l <= limit: self.assert_(len(self.env.account.containers(parms=p)) <= l) self.assert_status(200) else: self.assertRaises(ResponseError, self.env.account.containers, parms=p) self.assert_status(412) def testContainerListing(self): a = sorted([c.name for c in self.env.containers]) for format_type in [None, 'json', 'xml']: b = self.env.account.containers(parms={'format': format_type}) if isinstance(b[0], dict): b = [x['name'] for x in b] self.assertEqual(a, b) def testInvalidAuthToken(self): hdrs = {'X-Auth-Token': 'bogus_auth_token'} self.assertRaises(ResponseError, self.env.account.info, hdrs=hdrs) self.assert_status(401) def testLastContainerMarker(self): for format_type in [None, 'json', 'xml']: containers = self.env.account.containers({'format': format_type}) self.assertEqual(len(containers), len(self.env.containers)) self.assert_status(200) containers = self.env.account.containers( parms={'format': format_type, 'marker': containers[-1]}) self.assertEqual(len(containers), 0) if format_type is None: self.assert_status(204) else: self.assert_status(200) def testMarkerLimitContainerList(self): for format_type in [None, 'json', 'xml']: for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', 'abc123', 'mnop', 'xyz']: limit = random.randint(2, 9) containers = self.env.account.containers( parms={'format': format_type, 'marker': marker, 'limit': limit}) self.assert_(len(containers) <= limit) if containers: if isinstance(containers[0], dict): containers = [x['name'] for x in containers] self.assert_(locale.strcoll(containers[0], marker) > 0) def testContainersOrderedByName(self): for format_type in [None, 'json', 'xml']: containers = self.env.account.containers( parms={'format': format_type}) if isinstance(containers[0], dict): containers = [x['name'] for x in containers] self.assertEqual(sorted(containers, cmp=locale.strcoll), containers) class TestAccountUTF8(Base2, TestAccount): set_up = False class TestAccountNoContainersEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() class TestAccountNoContainers(Base): env = TestAccountNoContainersEnv set_up = False def testGetRequest(self): for format_type in [None, 'json', 'xml']: self.assert_(not self.env.account.containers( parms={'format': format_type})) if format_type is None: self.assert_status(204) else: self.assert_status(200) class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers): set_up = False class TestContainerEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) cls.file_count = 10 cls.file_size = 128 cls.files = list() for x in range(cls.file_count): file_item = cls.container.file(Utils.create_name()) file_item.write_random(cls.file_size) cls.files.append(file_item.name) class TestContainerDev(Base): env = TestContainerEnv set_up = False class TestContainerDevUTF8(Base2, TestContainerDev): set_up = False class TestContainer(Base): env = TestContainerEnv set_up = False def testContainerNameLimit(self): limit = load_constraint('max_container_name_length') for l in (limit - 100, limit - 10, limit - 1, limit, limit + 1, limit + 10, limit + 100): cont = self.env.account.container('a' * l) if l <= limit: self.assert_(cont.create()) self.assert_status(201) else: self.assert_(not cont.create()) self.assert_status(400) def testFileThenContainerDelete(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) file_item = cont.file(Utils.create_name()) self.assert_(file_item.write_random()) self.assert_(file_item.delete()) self.assert_status(204) self.assert_(file_item.name not in cont.files()) self.assert_(cont.delete()) self.assert_status(204) self.assert_(cont.name not in self.env.account.containers()) def testFileListingLimitMarkerPrefix(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) files = sorted([Utils.create_name() for x in xrange(10)]) for f in files: file_item = cont.file(f) self.assert_(file_item.write_random()) for i in xrange(len(files)): f = files[i] for j in xrange(1, len(files) - i): self.assert_(cont.files(parms={'limit': j, 'marker': f}) == files[i + 1: i + j + 1]) self.assert_(cont.files(parms={'marker': f}) == files[i + 1:]) self.assert_(cont.files(parms={'marker': f, 'prefix': f}) == []) self.assert_(cont.files(parms={'prefix': f}) == [f]) def testPrefixAndLimit(self): load_constraint('container_listing_limit') cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) prefix_file_count = 10 limit_count = 2 prefixs = ['alpha/', 'beta/', 'kappa/'] prefix_files = {} for prefix in prefixs: prefix_files[prefix] = [] for i in range(prefix_file_count): file_item = cont.file(prefix + Utils.create_name()) file_item.write() prefix_files[prefix].append(file_item.name) for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'prefix': prefix}) self.assertEqual(files, sorted(prefix_files[prefix])) for format_type in [None, 'json', 'xml']: for prefix in prefixs: files = cont.files(parms={'limit': limit_count, 'prefix': prefix}) self.assertEqual(len(files), limit_count) for file_item in files: self.assert_(file_item.startswith(prefix)) def testCreate(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) self.assert_status(201) self.assert_(cont.name in self.env.account.containers()) def testContainerFileListOnContainerThatDoesNotExist(self): for format_type in [None, 'json', 'xml']: container = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, container.files, parms={'format': format_type}) self.assert_status(404) def testUtf8Container(self): valid_utf8 = Utils.create_utf8_name() invalid_utf8 = valid_utf8[::-1] container = self.env.account.container(valid_utf8) self.assert_(container.create(cfg={'no_path_quote': True})) self.assert_(container.name in self.env.account.containers()) self.assertEqual(container.files(), []) self.assert_(container.delete()) container = self.env.account.container(invalid_utf8) self.assert_(not container.create(cfg={'no_path_quote': True})) self.assert_status(412) self.assertRaises(ResponseError, container.files, cfg={'no_path_quote': True}) self.assert_status(412) def testCreateOnExisting(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) self.assert_status(201) self.assert_(cont.create()) self.assert_status(202) def testSlashInName(self): if Utils.create_name == Utils.create_utf8_name: cont_name = list(unicode(Utils.create_name(), 'utf-8')) else: cont_name = list(Utils.create_name()) cont_name[random.randint(2, len(cont_name) - 2)] = '/' cont_name = ''.join(cont_name) if Utils.create_name == Utils.create_utf8_name: cont_name = cont_name.encode('utf-8') cont = self.env.account.container(cont_name) self.assert_(not cont.create(cfg={'no_path_quote': True}), 'created container with name %s' % (cont_name)) self.assert_status(404) self.assert_(cont.name not in self.env.account.containers()) def testDelete(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) self.assert_status(201) self.assert_(cont.delete()) self.assert_status(204) self.assert_(cont.name not in self.env.account.containers()) def testDeleteOnContainerThatDoesNotExist(self): cont = self.env.account.container(Utils.create_name()) self.assert_(not cont.delete()) self.assert_status(404) def testDeleteOnContainerWithFiles(self): cont = self.env.account.container(Utils.create_name()) self.assert_(cont.create()) file_item = cont.file(Utils.create_name()) file_item.write_random(self.env.file_size) self.assert_(file_item.name in cont.files()) self.assert_(not cont.delete()) self.assert_status(409) def testFileCreateInContainerThatDoesNotExist(self): file_item = File(self.env.conn, self.env.account, Utils.create_name(), Utils.create_name()) self.assertRaises(ResponseError, file_item.write) self.assert_status(404) def testLastFileMarker(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files({'format': format_type}) self.assertEqual(len(files), len(self.env.files)) self.assert_status(200) files = self.env.container.files( parms={'format': format_type, 'marker': files[-1]}) self.assertEqual(len(files), 0) if format_type is None: self.assert_status(204) else: self.assert_status(200) def testContainerFileList(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files(parms={'format': format_type}) self.assert_status(200) if isinstance(files[0], dict): files = [x['name'] for x in files] for file_item in self.env.files: self.assert_(file_item in files) for file_item in files: self.assert_(file_item in self.env.files) def testMarkerLimitFileList(self): for format_type in [None, 'json', 'xml']: for marker in ['0', 'A', 'I', 'R', 'Z', 'a', 'i', 'r', 'z', 'abc123', 'mnop', 'xyz']: limit = random.randint(2, self.env.file_count - 1) files = self.env.container.files(parms={'format': format_type, 'marker': marker, 'limit': limit}) if not files: continue if isinstance(files[0], dict): files = [x['name'] for x in files] self.assert_(len(files) <= limit) if files: if isinstance(files[0], dict): files = [x['name'] for x in files] self.assert_(locale.strcoll(files[0], marker) > 0) def testFileOrder(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files(parms={'format': format_type}) if isinstance(files[0], dict): files = [x['name'] for x in files] self.assertEqual(sorted(files, cmp=locale.strcoll), files) def testContainerInfo(self): info = self.env.container.info() self.assert_status(204) self.assertEqual(info['object_count'], self.env.file_count) self.assertEqual(info['bytes_used'], self.env.file_count * self.env.file_size) def testContainerInfoOnContainerThatDoesNotExist(self): container = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, container.info) self.assert_status(404) def testContainerFileListWithLimit(self): for format_type in [None, 'json', 'xml']: files = self.env.container.files(parms={'format': format_type, 'limit': 2}) self.assertEqual(len(files), 2) def testTooLongName(self): cont = self.env.account.container('x' * 257) self.assert_(not cont.create(), 'created container with name %s' % (cont.name)) self.assert_status(400) def testContainerExistenceCachingProblem(self): cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) self.assert_(cont.create()) cont.files() cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) self.assert_(cont.create()) file_item = cont.file(Utils.create_name()) file_item.write_random() class TestContainerUTF8(Base2, TestContainer): set_up = False class TestContainerPathsEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.file_size = 8 cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) cls.files = [ '/file1', '/file A', '/dir1/', '/dir2/', '/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/', '/dir1/subdir1/file2', '/dir1/subdir1/file3', '/dir1/subdir1/file4', '/dir1/subdir1/subsubdir1/', '/dir1/subdir1/subsubdir1/file5', '/dir1/subdir1/subsubdir1/file6', '/dir1/subdir1/subsubdir1/file7', '/dir1/subdir1/subsubdir1/file8', '/dir1/subdir1/subsubdir2/', '/dir1/subdir1/subsubdir2/file9', '/dir1/subdir1/subsubdir2/file0', 'file1', 'dir1/', 'dir2/', 'dir1/file2', 'dir1/subdir1/', 'dir1/subdir2/', 'dir1/subdir1/file2', 'dir1/subdir1/file3', 'dir1/subdir1/file4', 'dir1/subdir1/subsubdir1/', 'dir1/subdir1/subsubdir1/file5', 'dir1/subdir1/subsubdir1/file6', 'dir1/subdir1/subsubdir1/file7', 'dir1/subdir1/subsubdir1/file8', 'dir1/subdir1/subsubdir2/', 'dir1/subdir1/subsubdir2/file9', 'dir1/subdir1/subsubdir2/file0', 'dir1/subdir with spaces/', 'dir1/subdir with spaces/file B', 'dir1/subdir+with{whatever/', 'dir1/subdir+with{whatever/file D', ] stored_files = set() for f in cls.files: file_item = cls.container.file(f) if f.endswith('/'): file_item.write(hdrs={'Content-Type': 'application/directory'}) else: file_item.write_random(cls.file_size, hdrs={'Content-Type': 'application/directory'}) if (normalized_urls): nfile = '/'.join(filter(None, f.split('/'))) if (f[-1] == '/'): nfile += '/' stored_files.add(nfile) else: stored_files.add(f) cls.stored_files = sorted(stored_files) class TestContainerPaths(Base): env = TestContainerPathsEnv set_up = False def testTraverseContainer(self): found_files = [] found_dirs = [] def recurse_path(path, count=0): if count > 10: raise ValueError('too deep recursion') for file_item in self.env.container.files(parms={'path': path}): self.assert_(file_item.startswith(path)) if file_item.endswith('/'): recurse_path(file_item, count + 1) found_dirs.append(file_item) else: found_files.append(file_item) recurse_path('') for file_item in self.env.stored_files: if file_item.startswith('/'): self.assert_(file_item not in found_dirs) self.assert_(file_item not in found_files) elif file_item.endswith('/'): self.assert_(file_item in found_dirs) self.assert_(file_item not in found_files) else: self.assert_(file_item in found_files) self.assert_(file_item not in found_dirs) found_files = [] found_dirs = [] recurse_path('/') for file_item in self.env.stored_files: if not file_item.startswith('/'): self.assert_(file_item not in found_dirs) self.assert_(file_item not in found_files) elif file_item.endswith('/'): self.assert_(file_item in found_dirs) self.assert_(file_item not in found_files) else: self.assert_(file_item in found_files) self.assert_(file_item not in found_dirs) def testContainerListing(self): for format_type in (None, 'json', 'xml'): files = self.env.container.files(parms={'format': format_type}) if isinstance(files[0], dict): files = [str(x['name']) for x in files] self.assertEqual(files, self.env.stored_files) for format_type in ('json', 'xml'): for file_item in self.env.container.files(parms={'format': format_type}): self.assert_(int(file_item['bytes']) >= 0) self.assert_('last_modified' in file_item) if file_item['name'].endswith('/'): self.assertEqual(file_item['content_type'], 'application/directory') def testStructure(self): def assert_listing(path, file_list): files = self.env.container.files(parms={'path': path}) self.assertEqual(sorted(file_list, cmp=locale.strcoll), files) if not normalized_urls: assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) assert_listing('/dir1', ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) assert_listing('/dir1/', ['/dir1/file2', '/dir1/subdir1/', '/dir1/subdir2/']) assert_listing('/dir1/subdir1', ['/dir1/subdir1/subsubdir2/', '/dir1/subdir1/file2', '/dir1/subdir1/file3', '/dir1/subdir1/file4', '/dir1/subdir1/subsubdir1/']) assert_listing('/dir1/subdir2', []) assert_listing('', ['file1', 'dir1/', 'dir2/']) else: assert_listing('', ['file1', 'dir1/', 'dir2/', 'file A']) assert_listing('dir1', ['dir1/file2', 'dir1/subdir1/', 'dir1/subdir2/', 'dir1/subdir with spaces/', 'dir1/subdir+with{whatever/']) assert_listing('dir1/subdir1', ['dir1/subdir1/file4', 'dir1/subdir1/subsubdir2/', 'dir1/subdir1/file2', 'dir1/subdir1/file3', 'dir1/subdir1/subsubdir1/']) assert_listing('dir1/subdir1/subsubdir1', ['dir1/subdir1/subsubdir1/file7', 'dir1/subdir1/subsubdir1/file5', 'dir1/subdir1/subsubdir1/file8', 'dir1/subdir1/subsubdir1/file6']) assert_listing('dir1/subdir1/subsubdir1/', ['dir1/subdir1/subsubdir1/file7', 'dir1/subdir1/subsubdir1/file5', 'dir1/subdir1/subsubdir1/file8', 'dir1/subdir1/subsubdir1/file6']) assert_listing('dir1/subdir with spaces/', ['dir1/subdir with spaces/file B']) class TestFileEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) cls.file_size = 128 class TestFileDev(Base): env = TestFileEnv set_up = False class TestFileDevUTF8(Base2, TestFileDev): set_up = False class TestFile(Base): env = TestFileEnv set_up = False def testCopy(self): # makes sure to test encoded characters source_filename = 'dealde%2Fl04 011e%204c8df/flash.png' file_item = self.env.container.file(source_filename) metadata = {} for i in range(1): metadata[Utils.create_ascii_name()] = Utils.create_name() data = file_item.write_random() file_item.sync_metadata(metadata) dest_cont = self.env.account.container(Utils.create_name()) self.assert_(dest_cont.create()) # copy both from within and across containers for cont in (self.env.container, dest_cont): # copy both with and without initial slash for prefix in ('', '/'): dest_filename = Utils.create_name() file_item = self.env.container.file(source_filename) file_item.copy('%s%s' % (prefix, cont), dest_filename) self.assert_(dest_filename in cont.files()) file_item = cont.file(dest_filename) self.assert_(data == file_item.read()) self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) def testCopy404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) self.assert_(dest_cont.create()) for prefix in ('', '/'): # invalid source container source_cont = self.env.account.container(Utils.create_name()) file_item = source_cont.file(source_filename) self.assert_(not file_item.copy( '%s%s' % (prefix, self.env.container), Utils.create_name())) self.assert_status(404) self.assert_(not file_item.copy('%s%s' % (prefix, dest_cont), Utils.create_name())) self.assert_status(404) # invalid source object file_item = self.env.container.file(Utils.create_name()) self.assert_(not file_item.copy( '%s%s' % (prefix, self.env.container), Utils.create_name())) self.assert_status(404) self.assert_(not file_item.copy('%s%s' % (prefix, dest_cont), Utils.create_name())) self.assert_status(404) # invalid destination container file_item = self.env.container.file(source_filename) self.assert_(not file_item.copy( '%s%s' % (prefix, Utils.create_name()), Utils.create_name())) def testCopyNoDestinationHeader(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) file_item.write_random() file_item = self.env.container.file(source_filename) self.assert_(not file_item.copy(Utils.create_name(), Utils.create_name(), cfg={'no_destination': True})) self.assert_status(412) def testCopyDestinationSlashProblems(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) file_item.write_random() # no slash self.assert_(not file_item.copy(Utils.create_name(), Utils.create_name(), cfg={'destination': Utils.create_name()})) self.assert_status(412) def testCopyFromHeader(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) metadata = {} for i in range(1): metadata[Utils.create_ascii_name()] = Utils.create_name() file_item.metadata = metadata data = file_item.write_random() dest_cont = self.env.account.container(Utils.create_name()) self.assert_(dest_cont.create()) # copy both from within and across containers for cont in (self.env.container, dest_cont): # copy both with and without initial slash for prefix in ('', '/'): dest_filename = Utils.create_name() file_item = cont.file(dest_filename) file_item.write(hdrs={'X-Copy-From': '%s%s/%s' % ( prefix, self.env.container.name, source_filename)}) self.assert_(dest_filename in cont.files()) file_item = cont.file(dest_filename) self.assert_(data == file_item.read()) self.assert_(file_item.initialize()) self.assert_(metadata == file_item.metadata) def testCopyFromHeader404s(self): source_filename = Utils.create_name() file_item = self.env.container.file(source_filename) file_item.write_random() for prefix in ('', '/'): # invalid source container file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.write, hdrs={'X-Copy-From': '%s%s/%s' % (prefix, Utils.create_name(), source_filename)}) self.assert_status(404) # invalid source object file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.write, hdrs={'X-Copy-From': '%s%s/%s' % (prefix, self.env.container.name, Utils.create_name())}) self.assert_status(404) # invalid destination container dest_cont = self.env.account.container(Utils.create_name()) file_item = dest_cont.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.write, hdrs={'X-Copy-From': '%s%s/%s' % (prefix, self.env.container.name, source_filename)}) self.assert_status(404) def testNameLimit(self): limit = load_constraint('max_object_name_length') for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): file_item = self.env.container.file('a' * l) if l <= limit: self.assert_(file_item.write()) self.assert_status(201) else: self.assertRaises(ResponseError, file_item.write) self.assert_status(400) def testQuestionMarkInName(self): if Utils.create_name == Utils.create_ascii_name: file_name = list(Utils.create_name()) file_name[random.randint(2, len(file_name) - 2)] = '?' file_name = "".join(file_name) else: file_name = Utils.create_name(6) + '?' + Utils.create_name(6) file_item = self.env.container.file(file_name) self.assert_(file_item.write(cfg={'no_path_quote': True})) self.assert_(file_name not in self.env.container.files()) self.assert_(file_name.split('?')[0] in self.env.container.files()) def testDeleteThen404s(self): file_item = self.env.container.file(Utils.create_name()) self.assert_(file_item.write_random()) self.assert_status(201) self.assert_(file_item.delete()) self.assert_status(204) file_item.metadata = {Utils.create_ascii_name(): Utils.create_name()} for method in (file_item.info, file_item.read, file_item.sync_metadata, file_item.delete): self.assertRaises(ResponseError, method) self.assert_status(404) def testBlankMetadataName(self): file_item = self.env.container.file(Utils.create_name()) file_item.metadata = {'': Utils.create_name()} self.assertRaises(ResponseError, file_item.write_random) self.assert_status(400) def testMetadataNumberLimit(self): number_limit = load_constraint('max_meta_count') size_limit = load_constraint('max_meta_overall_size') for i in (number_limit - 10, number_limit - 1, number_limit, number_limit + 1, number_limit + 10, number_limit + 100): j = size_limit / (i * 2) size = 0 metadata = {} while len(metadata.keys()) < i: key = Utils.create_ascii_name() val = Utils.create_name() if len(key) > j: key = key[:j] val = val[:j] size += len(key) + len(val) metadata[key] = val file_item = self.env.container.file(Utils.create_name()) file_item.metadata = metadata if i <= number_limit: self.assert_(file_item.write()) self.assert_status(201) self.assert_(file_item.sync_metadata()) self.assert_status((201, 202)) else: self.assertRaises(ResponseError, file_item.write) self.assert_status(400) file_item.metadata = {} self.assert_(file_item.write()) self.assert_status(201) file_item.metadata = metadata self.assertRaises(ResponseError, file_item.sync_metadata) self.assert_status(400) def testContentTypeGuessing(self): file_types = {'wav': 'audio/x-wav', 'txt': 'text/plain', 'zip': 'application/zip'} container = self.env.account.container(Utils.create_name()) self.assert_(container.create()) for i in file_types.keys(): file_item = container.file(Utils.create_name() + '.' + i) file_item.write('', cfg={'no_content_type': True}) file_types_read = {} for i in container.files(parms={'format': 'json'}): file_types_read[i['name'].split('.')[1]] = i['content_type'] self.assertEqual(file_types, file_types_read) def testRangedGets(self): file_length = 10000 range_size = file_length / 10 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) for i in range(0, file_length, range_size): range_string = 'bytes=%d-%d' % (i, i + range_size - 1) hdrs = {'Range': range_string} self.assert_(data[i: i + range_size] == file_item.read(hdrs=hdrs), range_string) range_string = 'bytes=-%d' % (i) hdrs = {'Range': range_string} if i == 0: # RFC 2616 14.35.1 # "If a syntactically valid byte-range-set includes ... at # least one suffix-byte-range-spec with a NON-ZERO # suffix-length, then the byte-range-set is satisfiable. # Otherwise, the byte-range-set is unsatisfiable. self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(416) else: self.assertEqual(file_item.read(hdrs=hdrs), data[-i:]) range_string = 'bytes=%d-' % (i) hdrs = {'Range': range_string} self.assert_(file_item.read(hdrs=hdrs) == data[i - file_length:], range_string) range_string = 'bytes=%d-%d' % (file_length + 1000, file_length + 2000) hdrs = {'Range': range_string} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(416) range_string = 'bytes=%d-%d' % (file_length - 1000, file_length + 2000) hdrs = {'Range': range_string} self.assert_(file_item.read(hdrs=hdrs) == data[-1000:], range_string) hdrs = {'Range': '0-4'} self.assert_(file_item.read(hdrs=hdrs) == data, range_string) # RFC 2616 14.35.1 # "If the entity is shorter than the specified suffix-length, the # entire entity-body is used." range_string = 'bytes=-%d' % (file_length + 10) hdrs = {'Range': range_string} self.assert_(file_item.read(hdrs=hdrs) == data, range_string) def testRangedGetsWithLWSinHeader(self): #Skip this test until webob 1.2 can tolerate LWS in Range header. file_length = 10000 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) for r in ('BYTES=0-999', 'bytes = 0-999', 'BYTES = 0 - 999', 'bytes = 0 - 999', 'bytes=0 - 999', 'bytes=0-999 '): self.assert_(file_item.read(hdrs={'Range': r}) == data[0:1000]) def testFileSizeLimit(self): limit = load_constraint('max_file_size') tsecs = 3 for i in (limit - 100, limit - 10, limit - 1, limit, limit + 1, limit + 10, limit + 100): file_item = self.env.container.file(Utils.create_name()) if i <= limit: self.assert_(timeout(tsecs, file_item.write, cfg={'set_content_length': i})) else: self.assertRaises(ResponseError, timeout, tsecs, file_item.write, cfg={'set_content_length': i}) def testNoContentLengthForPut(self): file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.write, 'testing', cfg={'no_content_length': True}) self.assert_status(411) def testDelete(self): file_item = self.env.container.file(Utils.create_name()) file_item.write_random(self.env.file_size) self.assert_(file_item.name in self.env.container.files()) self.assert_(file_item.delete()) self.assert_(file_item.name not in self.env.container.files()) def testBadHeaders(self): file_length = 100 # no content type on puts should be ok file_item = self.env.container.file(Utils.create_name()) file_item.write_random(file_length, cfg={'no_content_type': True}) self.assert_status(201) # content length x self.assertRaises(ResponseError, file_item.write_random, file_length, hdrs={'Content-Length': 'X'}, cfg={'no_content_length': True}) self.assert_status(400) # bad request types #for req in ('LICK', 'GETorHEAD_base', 'container_info', # 'best_response'): for req in ('LICK', 'GETorHEAD_base'): self.env.account.conn.make_request(req) self.assert_status(405) # bad range headers self.assert_(len(file_item.read(hdrs={'Range': 'parsecs=8-12'})) == file_length) self.assert_status(200) def testMetadataLengthLimits(self): key_limit = load_constraint('max_meta_name_length') value_limit = load_constraint('max_meta_value_length') lengths = [[key_limit, value_limit], [key_limit, value_limit + 1], [key_limit + 1, value_limit], [key_limit, 0], [key_limit, value_limit * 10], [key_limit * 10, value_limit]] for l in lengths: metadata = {'a' * l[0]: 'b' * l[1]} file_item = self.env.container.file(Utils.create_name()) file_item.metadata = metadata if l[0] <= key_limit and l[1] <= value_limit: self.assert_(file_item.write()) self.assert_status(201) self.assert_(file_item.sync_metadata()) else: self.assertRaises(ResponseError, file_item.write) self.assert_status(400) file_item.metadata = {} self.assert_(file_item.write()) self.assert_status(201) file_item.metadata = metadata self.assertRaises(ResponseError, file_item.sync_metadata) self.assert_status(400) def testEtagWayoff(self): file_item = self.env.container.file(Utils.create_name()) hdrs = {'etag': 'reallylonganddefinitelynotavalidetagvalue'} self.assertRaises(ResponseError, file_item.write_random, hdrs=hdrs) self.assert_status(422) def testFileCreate(self): for i in range(10): file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random() self.assert_status(201) self.assert_(data == file_item.read()) self.assert_status(200) def testHead(self): file_name = Utils.create_name() content_type = Utils.create_name() file_item = self.env.container.file(file_name) file_item.content_type = content_type file_item.write_random(self.env.file_size) md5 = file_item.md5 file_item = self.env.container.file(file_name) info = file_item.info() self.assert_status(200) self.assertEqual(info['content_length'], self.env.file_size) self.assertEqual(info['etag'], md5) self.assertEqual(info['content_type'], content_type) self.assert_('last_modified' in info) def testDeleteOfFileThatDoesNotExist(self): # in container that exists file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.delete) self.assert_status(404) # in container that does not exist container = self.env.account.container(Utils.create_name()) file_item = container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.delete) self.assert_status(404) def testHeadOnFileThatDoesNotExist(self): # in container that exists file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.info) self.assert_status(404) # in container that does not exist container = self.env.account.container(Utils.create_name()) file_item = container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.info) self.assert_status(404) def testMetadataOnPost(self): file_item = self.env.container.file(Utils.create_name()) file_item.write_random(self.env.file_size) for i in range(10): metadata = {} for j in range(10): metadata[Utils.create_ascii_name()] = Utils.create_name() file_item.metadata = metadata self.assert_(file_item.sync_metadata()) self.assert_status((201, 202)) file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) self.assertEqual(file_item.metadata, metadata) def testGetContentType(self): file_name = Utils.create_name() content_type = Utils.create_name() file_item = self.env.container.file(file_name) file_item.content_type = content_type file_item.write_random() file_item = self.env.container.file(file_name) file_item.read() self.assertEqual(content_type, file_item.content_type) def testGetOnFileThatDoesNotExist(self): # in container that exists file_item = self.env.container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.read) self.assert_status(404) # in container that does not exist container = self.env.account.container(Utils.create_name()) file_item = container.file(Utils.create_name()) self.assertRaises(ResponseError, file_item.read) self.assert_status(404) def testPostOnFileThatDoesNotExist(self): # in container that exists file_item = self.env.container.file(Utils.create_name()) file_item.metadata['Field'] = 'Value' self.assertRaises(ResponseError, file_item.sync_metadata) self.assert_status(404) # in container that does not exist container = self.env.account.container(Utils.create_name()) file_item = container.file(Utils.create_name()) file_item.metadata['Field'] = 'Value' self.assertRaises(ResponseError, file_item.sync_metadata) self.assert_status(404) def testMetadataOnPut(self): for i in range(10): metadata = {} for j in range(10): metadata[Utils.create_ascii_name()] = Utils.create_name() file_item = self.env.container.file(Utils.create_name()) file_item.metadata = metadata file_item.write_random(self.env.file_size) file_item = self.env.container.file(file_item.name) self.assert_(file_item.initialize()) self.assert_status(200) self.assertEqual(file_item.metadata, metadata) def testSerialization(self): container = self.env.account.container(Utils.create_name()) self.assert_(container.create()) files = [] for i in (0, 1, 10, 100, 1000, 10000): files.append({'name': Utils.create_name(), 'content_type': Utils.create_name(), 'bytes': i}) write_time = time.time() for f in files: file_item = container.file(f['name']) file_item.content_type = f['content_type'] file_item.write_random(f['bytes']) f['hash'] = file_item.md5 f['json'] = False f['xml'] = False write_time = time.time() - write_time for format_type in ['json', 'xml']: for file_item in container.files(parms={'format': format_type}): found = False for f in files: if f['name'] != file_item['name']: continue self.assertEqual(file_item['content_type'], f['content_type']) self.assertEqual(int(file_item['bytes']), f['bytes']) d = datetime.strptime( file_item['last_modified'].split('.')[0], "%Y-%m-%dT%H:%M:%S") lm = time.mktime(d.timetuple()) if 'last_modified' in f: self.assertEqual(f['last_modified'], lm) else: f['last_modified'] = lm f[format_type] = True found = True self.assert_(found, 'Unexpected file %s found in ' '%s listing' % (file_item['name'], format_type)) headers = dict(self.env.conn.response.getheaders()) if format_type == 'json': self.assertEqual(headers['content-type'], 'application/json; charset=utf-8') elif format_type == 'xml': self.assertEqual(headers['content-type'], 'application/xml; charset=utf-8') lm_diff = max([f['last_modified'] for f in files]) -\ min([f['last_modified'] for f in files]) self.assert_(lm_diff < write_time + 1, 'Diff in last ' 'modified times should be less than time to write files') for f in files: for format_type in ['json', 'xml']: self.assert_(f[format_type], 'File %s not found in %s listing' % (f['name'], format_type)) def testStackedOverwrite(self): file_item = self.env.container.file(Utils.create_name()) for i in range(1, 11): data = file_item.write_random(512) file_item.write(data) self.assert_(file_item.read() == data) def testTooLongName(self): file_item = self.env.container.file('x' * 1025) self.assertRaises(ResponseError, file_item.write) self.assert_status(400) def testZeroByteFile(self): file_item = self.env.container.file(Utils.create_name()) self.assert_(file_item.write('')) self.assert_(file_item.name in self.env.container.files()) self.assert_(file_item.read() == '') def testEtagResponse(self): file_item = self.env.container.file(Utils.create_name()) data = StringIO.StringIO(file_item.write_random(512)) etag = File.compute_md5sum(data) headers = dict(self.env.conn.response.getheaders()) self.assert_('etag' in headers.keys()) header_etag = headers['etag'].strip('"') self.assertEqual(etag, header_etag) def testChunkedPut(self): if (web_front_end == 'apache2'): raise SkipTest() data = File.random_data(10000) etag = File.compute_md5sum(data) for i in (1, 10, 100, 1000): file_item = self.env.container.file(Utils.create_name()) for j in chunks(data, i): file_item.chunked_write(j) self.assert_(file_item.chunked_write()) self.assert_(data == file_item.read()) info = file_item.info() self.assertEqual(etag, info['etag']) class TestFileUTF8(Base2, TestFile): set_up = False class TestDloEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) # avoid getting a prefix that stops halfway through an encoded # character prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") cls.segment_prefix = prefix for letter in ('a', 'b', 'c', 'd', 'e'): file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter)) file_item.write(letter * 10) file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter)) file_item.write(letter.upper() * 10) man1 = cls.container.file("man1") man1.write('man1-contents', hdrs={"X-Object-Manifest": "%s/%s/seg_lower" % (cls.container.name, prefix)}) man1 = cls.container.file("man2") man1.write('man2-contents', hdrs={"X-Object-Manifest": "%s/%s/seg_upper" % (cls.container.name, prefix)}) manall = cls.container.file("manall") manall.write('manall-contents', hdrs={"X-Object-Manifest": "%s/%s/seg" % (cls.container.name, prefix)}) class TestDlo(Base): env = TestDloEnv set_up = False def test_get_manifest(self): file_item = self.env.container.file('man1') file_contents = file_item.read() self.assertEqual( file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") file_item = self.env.container.file('man2') file_contents = file_item.read() self.assertEqual( file_contents, "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE") file_item = self.env.container.file('manall') file_contents = file_item.read() self.assertEqual( file_contents, ("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" + "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")) def test_get_manifest_document_itself(self): file_item = self.env.container.file('man1') file_contents = file_item.read(parms={'multipart-manifest': 'get'}) self.assertEqual(file_contents, "man1-contents") def test_get_range(self): file_item = self.env.container.file('man1') file_contents = file_item.read(size=25, offset=8) self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd") file_contents = file_item.read(size=1, offset=47) self.assertEqual(file_contents, "e") def test_get_range_out_of_range(self): file_item = self.env.container.file('man1') self.assertRaises(ResponseError, file_item.read, size=7, offset=50) self.assert_status(416) def test_copy(self): # Adding a new segment, copying the manifest, and then deleting the # segment proves that the new object is really the concatenated # segments and not just a manifest. f_segment = self.env.container.file("%s/seg_lowerf" % (self.env.segment_prefix)) f_segment.write('ffffffffff') try: man1_item = self.env.container.file('man1') man1_item.copy(self.env.container.name, "copied-man1") finally: # try not to leave this around for other tests to stumble over f_segment.delete() file_item = self.env.container.file('copied-man1') file_contents = file_item.read() self.assertEqual( file_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") def test_copy_manifest(self): # Copying the manifest should result in another manifest try: man1_item = self.env.container.file('man1') man1_item.copy(self.env.container.name, "copied-man1", parms={'multipart-manifest': 'get'}) copied = self.env.container.file("copied-man1") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) self.assertEqual(copied_contents, "man1-contents") copied_contents = copied.read() self.assertEqual( copied_contents, "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") finally: # try not to leave this around for other tests to stumble over self.env.container.file("copied-man1").delete() def test_dlo_if_match_get(self): manifest = self.env.container.file("man1") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.read, hdrs={'If-Match': 'not-%s' % etag}) self.assert_status(412) manifest.read(hdrs={'If-Match': etag}) self.assert_status(200) def test_dlo_if_none_match_get(self): manifest = self.env.container.file("man1") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.read, hdrs={'If-None-Match': etag}) self.assert_status(304) manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) self.assert_status(200) def test_dlo_if_match_head(self): manifest = self.env.container.file("man1") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.info, hdrs={'If-Match': 'not-%s' % etag}) self.assert_status(412) manifest.info(hdrs={'If-Match': etag}) self.assert_status(200) def test_dlo_if_none_match_head(self): manifest = self.env.container.file("man1") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.info, hdrs={'If-None-Match': etag}) self.assert_status(304) manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) self.assert_status(200) class TestDloUTF8(Base2, TestDlo): set_up = False class TestFileComparisonEnv(object): @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) cls.file_count = 20 cls.file_size = 128 cls.files = list() for x in range(cls.file_count): file_item = cls.container.file(Utils.create_name()) file_item.write_random(cls.file_size) cls.files.append(file_item) cls.time_old_f1 = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(time.time() - 86400)) cls.time_old_f2 = time.strftime("%A, %d-%b-%y %H:%M:%S GMT", time.gmtime(time.time() - 86400)) cls.time_old_f3 = time.strftime("%a %b %d %H:%M:%S %Y", time.gmtime(time.time() - 86400)) cls.time_new = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(time.time() + 86400)) class TestFileComparison(Base): env = TestFileComparisonEnv set_up = False def testIfMatch(self): for file_item in self.env.files: hdrs = {'If-Match': file_item.md5} self.assert_(file_item.read(hdrs=hdrs)) hdrs = {'If-Match': 'bogus'} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) def testIfNoneMatch(self): for file_item in self.env.files: hdrs = {'If-None-Match': 'bogus'} self.assert_(file_item.read(hdrs=hdrs)) hdrs = {'If-None-Match': file_item.md5} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) def testIfModifiedSince(self): for file_item in self.env.files: hdrs = {'If-Modified-Since': self.env.time_old_f1} self.assert_(file_item.read(hdrs=hdrs)) hdrs = {'If-Modified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(304) def testIfUnmodifiedSince(self): for file_item in self.env.files: hdrs = {'If-Unmodified-Since': self.env.time_new} self.assert_(file_item.read(hdrs=hdrs)) hdrs = {'If-Unmodified-Since': self.env.time_old_f2} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) def testIfMatchAndUnmodified(self): for file_item in self.env.files: hdrs = {'If-Match': file_item.md5, 'If-Unmodified-Since': self.env.time_new} self.assert_(file_item.read(hdrs=hdrs)) hdrs = {'If-Match': 'bogus', 'If-Unmodified-Since': self.env.time_new} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) hdrs = {'If-Match': file_item.md5, 'If-Unmodified-Since': self.env.time_old_f3} self.assertRaises(ResponseError, file_item.read, hdrs=hdrs) self.assert_status(412) def testLastModified(self): file_name = Utils.create_name() content_type = Utils.create_name() file = self.env.container.file(file_name) file.content_type = content_type resp = file.write_random_return_resp(self.env.file_size) put_last_modified = resp.getheader('last-modified') file = self.env.container.file(file_name) info = file.info() self.assert_('last_modified' in info) last_modified = info['last_modified'] self.assertEqual(put_last_modified, info['last_modified']) hdrs = {'If-Modified-Since': last_modified} self.assertRaises(ResponseError, file.read, hdrs=hdrs) self.assert_status(304) hdrs = {'If-Unmodified-Since': last_modified} self.assert_(file.read(hdrs=hdrs)) class TestFileComparisonUTF8(Base2, TestFileComparison): set_up = False class TestSloEnv(object): slo_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() if cls.slo_enabled is None: cluster_info = cls.conn.cluster_info() cls.slo_enabled = 'slo' in cluster_info if not cls.slo_enabled: return cls.account = Account(cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) seg_info = {} for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024), ('c', 1024 * 1024), ('d', 1024 * 1024), ('e', 1)): seg_name = "seg_%s" % letter file_item = cls.container.file(seg_name) file_item.write(letter * size) seg_info[seg_name] = { 'size_bytes': size, 'etag': file_item.md5, 'path': '/%s/%s' % (cls.container.name, seg_name)} file_item = cls.container.file("manifest-abcde") file_item.write( json.dumps([seg_info['seg_a'], seg_info['seg_b'], seg_info['seg_c'], seg_info['seg_d'], seg_info['seg_e']]), parms={'multipart-manifest': 'put'}) file_item = cls.container.file('manifest-cd') cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']]) file_item.write(cd_json, parms={'multipart-manifest': 'put'}) cd_etag = hashlib.md5(seg_info['seg_c']['etag'] + seg_info['seg_d']['etag']).hexdigest() file_item = cls.container.file("manifest-bcd-submanifest") file_item.write( json.dumps([seg_info['seg_b'], {'etag': cd_etag, 'size_bytes': (seg_info['seg_c']['size_bytes'] + seg_info['seg_d']['size_bytes']), 'path': '/%s/%s' % (cls.container.name, 'manifest-cd')}]), parms={'multipart-manifest': 'put'}) bcd_submanifest_etag = hashlib.md5( seg_info['seg_b']['etag'] + cd_etag).hexdigest() file_item = cls.container.file("manifest-abcde-submanifest") file_item.write( json.dumps([ seg_info['seg_a'], {'etag': bcd_submanifest_etag, 'size_bytes': (seg_info['seg_b']['size_bytes'] + seg_info['seg_c']['size_bytes'] + seg_info['seg_d']['size_bytes']), 'path': '/%s/%s' % (cls.container.name, 'manifest-bcd-submanifest')}, seg_info['seg_e']]), parms={'multipart-manifest': 'put'}) class TestSlo(Base): env = TestSloEnv set_up = False def setUp(self): super(TestSlo, self).setUp() if self.env.slo_enabled is False: raise SkipTest("SLO not enabled") elif self.env.slo_enabled is not True: # just some sanity checking raise Exception( "Expected slo_enabled to be True/False, got %r" % (self.env.slo_enabled,)) def test_slo_get_simple_manifest(self): file_item = self.env.container.file('manifest-abcde') file_contents = file_item.read() self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) self.assertEqual('a', file_contents[0]) self.assertEqual('a', file_contents[1024 * 1024 - 1]) self.assertEqual('b', file_contents[1024 * 1024]) self.assertEqual('d', file_contents[-2]) self.assertEqual('e', file_contents[-1]) def test_slo_get_nested_manifest(self): file_item = self.env.container.file('manifest-abcde-submanifest') file_contents = file_item.read() self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) self.assertEqual('a', file_contents[0]) self.assertEqual('a', file_contents[1024 * 1024 - 1]) self.assertEqual('b', file_contents[1024 * 1024]) self.assertEqual('d', file_contents[-2]) self.assertEqual('e', file_contents[-1]) def test_slo_ranged_get(self): file_item = self.env.container.file('manifest-abcde') file_contents = file_item.read(size=1024 * 1024 + 2, offset=1024 * 1024 - 1) self.assertEqual('a', file_contents[0]) self.assertEqual('b', file_contents[1]) self.assertEqual('b', file_contents[-2]) self.assertEqual('c', file_contents[-1]) def test_slo_ranged_submanifest(self): file_item = self.env.container.file('manifest-abcde-submanifest') file_contents = file_item.read(size=1024 * 1024 + 2, offset=1024 * 1024 * 2 - 1) self.assertEqual('b', file_contents[0]) self.assertEqual('c', file_contents[1]) self.assertEqual('c', file_contents[-2]) self.assertEqual('d', file_contents[-1]) def test_slo_etag_is_hash_of_etags(self): expected_hash = hashlib.md5() expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest()) expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest()) expected_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest()) expected_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest()) expected_hash.update(hashlib.md5('e').hexdigest()) expected_etag = expected_hash.hexdigest() file_item = self.env.container.file('manifest-abcde') self.assertEqual(expected_etag, file_item.info()['etag']) def test_slo_etag_is_hash_of_etags_submanifests(self): def hd(x): return hashlib.md5(x).hexdigest() expected_etag = hd(hd('a' * 1024 * 1024) + hd(hd('b' * 1024 * 1024) + hd(hd('c' * 1024 * 1024) + hd('d' * 1024 * 1024))) + hd('e')) file_item = self.env.container.file('manifest-abcde-submanifest') self.assertEqual(expected_etag, file_item.info()['etag']) def test_slo_etag_mismatch(self): file_item = self.env.container.file("manifest-a-bad-etag") try: file_item.write( json.dumps([{ 'size_bytes': 1024 * 1024, 'etag': 'not it', 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), parms={'multipart-manifest': 'put'}) except ResponseError as err: self.assertEqual(400, err.status) else: self.fail("Expected ResponseError but didn't get it") def test_slo_size_mismatch(self): file_item = self.env.container.file("manifest-a-bad-size") try: file_item.write( json.dumps([{ 'size_bytes': 1024 * 1024 - 1, 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), parms={'multipart-manifest': 'put'}) except ResponseError as err: self.assertEqual(400, err.status) else: self.fail("Expected ResponseError but didn't get it") def test_slo_copy(self): file_item = self.env.container.file("manifest-abcde") file_item.copy(self.env.container.name, "copied-abcde") copied = self.env.container.file("copied-abcde") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) def test_slo_copy_the_manifest(self): file_item = self.env.container.file("manifest-abcde") file_item.copy(self.env.container.name, "copied-abcde-manifest-only", parms={'multipart-manifest': 'get'}) copied = self.env.container.file("copied-abcde-manifest-only") copied_contents = copied.read(parms={'multipart-manifest': 'get'}) try: json.loads(copied_contents) except ValueError: self.fail("COPY didn't copy the manifest (invalid json on GET)") def test_slo_get_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_body = manifest.read(parms={'multipart-manifest': 'get'}) self.assertEqual('application/json; charset=utf-8', manifest.content_type) try: json.loads(got_body) except ValueError: self.fail("GET with multipart-manifest=get got invalid json") def test_slo_head_the_manifest(self): manifest = self.env.container.file("manifest-abcde") got_info = manifest.info(parms={'multipart-manifest': 'get'}) self.assertEqual('application/json; charset=utf-8', got_info['content_type']) def test_slo_if_match_get(self): manifest = self.env.container.file("manifest-abcde") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.read, hdrs={'If-Match': 'not-%s' % etag}) self.assert_status(412) manifest.read(hdrs={'If-Match': etag}) self.assert_status(200) def test_slo_if_none_match_get(self): manifest = self.env.container.file("manifest-abcde") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.read, hdrs={'If-None-Match': etag}) self.assert_status(304) manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) self.assert_status(200) def test_slo_if_match_head(self): manifest = self.env.container.file("manifest-abcde") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.info, hdrs={'If-Match': 'not-%s' % etag}) self.assert_status(412) manifest.info(hdrs={'If-Match': etag}) self.assert_status(200) def test_slo_if_none_match_head(self): manifest = self.env.container.file("manifest-abcde") etag = manifest.info()['etag'] self.assertRaises(ResponseError, manifest.info, hdrs={'If-None-Match': etag}) self.assert_status(304) manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) self.assert_status(200) class TestSloUTF8(Base2, TestSlo): set_up = False class TestObjectVersioningEnv(object): versioning_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() cls.account = Account(cls.conn, config.get('account', config['username'])) # avoid getting a prefix that stops halfway through an encoded # character prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") cls.versions_container = cls.account.container(prefix + "-versions") if not cls.versions_container.create(): raise ResponseError(cls.conn.response) cls.container = cls.account.container(prefix + "-objs") if not cls.container.create( hdrs={'X-Versions-Location': cls.versions_container.name}): raise ResponseError(cls.conn.response) container_info = cls.container.info() # if versioning is off, then X-Versions-Location won't persist cls.versioning_enabled = 'versions' in container_info class TestObjectVersioning(Base): env = TestObjectVersioningEnv set_up = False def setUp(self): super(TestObjectVersioning, self).setUp() if self.env.versioning_enabled is False: raise SkipTest("Object versioning not enabled") elif self.env.versioning_enabled is not True: # just some sanity checking raise Exception( "Expected versioning_enabled to be True/False, got %r" % (self.env.versioning_enabled,)) def test_overwriting(self): container = self.env.container versions_container = self.env.versions_container obj_name = Utils.create_name() versioned_obj = container.file(obj_name) versioned_obj.write("aaaaa") self.assertEqual(0, versions_container.info()['object_count']) versioned_obj.write("bbbbb") # the old version got saved off self.assertEqual(1, versions_container.info()['object_count']) versioned_obj_name = versions_container.files()[0] self.assertEqual( "aaaaa", versions_container.file(versioned_obj_name).read()) # if we overwrite it again, there are two versions versioned_obj.write("ccccc") self.assertEqual(2, versions_container.info()['object_count']) # as we delete things, the old contents return self.assertEqual("ccccc", versioned_obj.read()) versioned_obj.delete() self.assertEqual("bbbbb", versioned_obj.read()) versioned_obj.delete() self.assertEqual("aaaaa", versioned_obj.read()) versioned_obj.delete() self.assertRaises(ResponseError, versioned_obj.read) class TestObjectVersioningUTF8(Base2, TestObjectVersioning): set_up = False class TestTempurlEnv(object): tempurl_enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() if cls.tempurl_enabled is None: cluster_info = cls.conn.cluster_info() cls.tempurl_enabled = 'tempurl' in cluster_info if not cls.tempurl_enabled: return cls.tempurl_methods = cluster_info['tempurl']['methods'] cls.tempurl_key = Utils.create_name() cls.tempurl_key2 = Utils.create_name() cls.account = Account( cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.account.update_metadata({ 'temp-url-key': cls.tempurl_key, 'temp-url-key-2': cls.tempurl_key2 }) cls.container = cls.account.container(Utils.create_name()) if not cls.container.create(): raise ResponseError(cls.conn.response) cls.obj = cls.container.file(Utils.create_name()) cls.obj.write("obj contents") cls.other_obj = cls.container.file(Utils.create_name()) cls.other_obj.write("other obj contents") class TestTempurl(Base): env = TestTempurlEnv set_up = False def setUp(self): super(TestTempurl, self).setUp() if self.env.tempurl_enabled is False: raise SkipTest("TempURL not enabled") elif self.env.tempurl_enabled is not True: # just some sanity checking raise Exception( "Expected tempurl_enabled to be True/False, got %r" % (self.env.tempurl_enabled,)) expires = int(time.time()) + 86400 sig = self.tempurl_sig( 'GET', expires, self.env.conn.make_path(self.env.obj.path), self.env.tempurl_key) self.obj_tempurl_parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} def tempurl_sig(self, method, expires, path, key): return hmac.new( key, '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), hashlib.sha1).hexdigest() def test_GET(self): contents = self.env.obj.read( parms=self.obj_tempurl_parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") # GET tempurls also allow HEAD requests self.assert_(self.env.obj.info(parms=self.obj_tempurl_parms, cfg={'no_auth_token': True})) def test_GET_with_key_2(self): expires = int(time.time()) + 86400 sig = self.tempurl_sig( 'GET', expires, self.env.conn.make_path(self.env.obj.path), self.env.tempurl_key2) parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") def test_PUT(self): new_obj = self.env.container.file(Utils.create_name()) expires = int(time.time()) + 86400 sig = self.tempurl_sig( 'PUT', expires, self.env.conn.make_path(new_obj.path), self.env.tempurl_key) put_parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} new_obj.write('new obj contents', parms=put_parms, cfg={'no_auth_token': True}) self.assertEqual(new_obj.read(), "new obj contents") # PUT tempurls also allow HEAD requests self.assert_(new_obj.info(parms=put_parms, cfg={'no_auth_token': True})) def test_HEAD(self): expires = int(time.time()) + 86400 sig = self.tempurl_sig( 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), self.env.tempurl_key) head_parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} self.assert_(self.env.obj.info(parms=head_parms, cfg={'no_auth_token': True})) # HEAD tempurls don't allow PUT or GET requests, despite the fact that # PUT and GET tempurls both allow HEAD requests self.assertRaises(ResponseError, self.env.other_obj.read, cfg={'no_auth_token': True}, parms=self.obj_tempurl_parms) self.assert_status([401]) self.assertRaises(ResponseError, self.env.other_obj.write, 'new contents', cfg={'no_auth_token': True}, parms=self.obj_tempurl_parms) self.assert_status([401]) def test_different_object(self): contents = self.env.obj.read( parms=self.obj_tempurl_parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") self.assertRaises(ResponseError, self.env.other_obj.read, cfg={'no_auth_token': True}, parms=self.obj_tempurl_parms) self.assert_status([401]) def test_changing_sig(self): contents = self.env.obj.read( parms=self.obj_tempurl_parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") parms = self.obj_tempurl_parms.copy() if parms['temp_url_sig'][0] == 'a': parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] else: parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] self.assertRaises(ResponseError, self.env.obj.read, cfg={'no_auth_token': True}, parms=parms) self.assert_status([401]) def test_changing_expires(self): contents = self.env.obj.read( parms=self.obj_tempurl_parms, cfg={'no_auth_token': True}) self.assertEqual(contents, "obj contents") parms = self.obj_tempurl_parms.copy() if parms['temp_url_expires'][-1] == '0': parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' else: parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' self.assertRaises(ResponseError, self.env.obj.read, cfg={'no_auth_token': True}, parms=parms) self.assert_status([401]) class TestTempurlUTF8(Base2, TestTempurl): set_up = False class TestSloTempurlEnv(object): enabled = None # tri-state: None initially, then True/False @classmethod def setUp(cls): cls.conn = Connection(config) cls.conn.authenticate() if cls.enabled is None: cluster_info = cls.conn.cluster_info() cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info cls.tempurl_key = Utils.create_name() cls.account = Account( cls.conn, config.get('account', config['username'])) cls.account.delete_containers() cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) cls.manifest_container = cls.account.container(Utils.create_name()) cls.segments_container = cls.account.container(Utils.create_name()) if not cls.manifest_container.create(): raise ResponseError(cls.conn.response) if not cls.segments_container.create(): raise ResponseError(cls.conn.response) seg1 = cls.segments_container.file(Utils.create_name()) seg1.write('1' * 1024 * 1024) seg2 = cls.segments_container.file(Utils.create_name()) seg2.write('2' * 1024 * 1024) cls.manifest_data = [{'size_bytes': 1024 * 1024, 'etag': seg1.md5, 'path': '/%s/%s' % (cls.segments_container.name, seg1.name)}, {'size_bytes': 1024 * 1024, 'etag': seg2.md5, 'path': '/%s/%s' % (cls.segments_container.name, seg2.name)}] cls.manifest = cls.manifest_container.file(Utils.create_name()) cls.manifest.write( json.dumps(cls.manifest_data), parms={'multipart-manifest': 'put'}) class TestSloTempurl(Base): env = TestSloTempurlEnv set_up = False def setUp(self): super(TestSloTempurl, self).setUp() if self.env.enabled is False: raise SkipTest("TempURL and SLO not both enabled") elif self.env.enabled is not True: # just some sanity checking raise Exception( "Expected enabled to be True/False, got %r" % (self.env.enabled,)) def tempurl_sig(self, method, expires, path, key): return hmac.new( key, '%s\n%s\n%s' % (method, expires, urllib.unquote(path)), hashlib.sha1).hexdigest() def test_GET(self): expires = int(time.time()) + 86400 sig = self.tempurl_sig( 'GET', expires, self.env.conn.make_path(self.env.manifest.path), self.env.tempurl_key) parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} contents = self.env.manifest.read( parms=parms, cfg={'no_auth_token': True}) self.assertEqual(len(contents), 2 * 1024 * 1024) # GET tempurls also allow HEAD requests self.assert_(self.env.manifest.info( parms=parms, cfg={'no_auth_token': True})) class TestSloTempurlUTF8(Base2, TestSloTempurl): set_up = False if __name__ == '__main__': unittest.main() swift-1.13.1/test/functional/test_object.py0000775000175400017540000011233612323703614022101 0ustar jenkinsjenkins00000000000000#!/usr/bin/python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest from nose import SkipTest from uuid import uuid4 from swift.common.utils import json from swift_testing import check_response, retry, skip, skip3, \ swift_test_perm, web_front_end, requires_acls, swift_test_user class TestObject(unittest.TestCase): def setUp(self): if skip: raise SkipTest self.container = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', parsed.path + '/' + self.container, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) self.obj = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, self.obj), 'test', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) def tearDown(self): if skip: raise SkipTest def delete(url, token, parsed, conn, obj): conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container, obj), '', {'X-Auth-Token': token}) return check_response(conn) # get list of objects in container def list(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(list) object_listing = resp.read() self.assertEqual(resp.status, 200) # iterate over object listing and delete all objects for obj in object_listing.splitlines(): resp = retry(delete, obj) resp.read() self.assertEqual(resp.status, 204) # delete the container def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.container, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) def test_if_none_match(self): def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, 'if_none_match_test'), '', {'X-Auth-Token': token, 'Content-Length': '0', 'If-None-Match': '*'}) return check_response(conn) resp = retry(put) resp.read() self.assertEquals(resp.status, 201) resp = retry(put) resp.read() self.assertEquals(resp.status, 412) def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, 'if_none_match_test'), '', {'X-Auth-Token': token, 'Content-Length': '0', 'If-None-Match': 'somethingelse'}) return check_response(conn) resp = retry(put) resp.read() self.assertEquals(resp.status, 400) def test_copy_object(self): if skip: raise SkipTest source = '%s/%s' % (self.container, self.obj) dest = '%s/%s' % (self.container, 'test_copy') # get contents of source def get_source(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, source), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get_source) source_contents = resp.read() self.assertEqual(resp.status, 200) self.assertEqual(source_contents, 'test') # copy source to dest with X-Copy-From def put(url, token, parsed, conn): conn.request('PUT', '%s/%s' % (parsed.path, dest), '', {'X-Auth-Token': token, 'Content-Length': '0', 'X-Copy-From': source}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # contents of dest should be the same as source def get_dest(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, dest), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get_dest) dest_contents = resp.read() self.assertEqual(resp.status, 200) self.assertEqual(dest_contents, source_contents) # delete the copy def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s' % (parsed.path, dest), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) # verify dest does not exist resp = retry(get_dest) resp.read() self.assertEqual(resp.status, 404) # copy source to dest with COPY def copy(url, token, parsed, conn): conn.request('COPY', '%s/%s' % (parsed.path, source), '', {'X-Auth-Token': token, 'Destination': dest}) return check_response(conn) resp = retry(copy) resp.read() self.assertEqual(resp.status, 201) # contents of dest should be the same as source resp = retry(get_dest) dest_contents = resp.read() self.assertEqual(resp.status, 200) self.assertEqual(dest_contents, source_contents) # delete the copy resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) def test_public_object(self): if skip: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', '%s/%s/%s' % (parsed.path, self.container, self.obj)) return check_response(conn) try: resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: self.assert_(str(err).startswith('No result after ')) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', {'X-Auth-Token': token, 'X-Container-Read': '.r:*'}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) resp = retry(get) resp.read() self.assertEqual(resp.status, 200) def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', {'X-Auth-Token': token, 'X-Container-Read': ''}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) try: resp = retry(get) raise Exception('Should not have been able to GET') except Exception as err: self.assert_(str(err).startswith('No result after ')) def test_private_object(self): if skip or skip3: raise SkipTest # Ensure we can't access the object with the third account def get(url, token, parsed, conn): conn.request('GET', '%s/%s/%s' % ( parsed.path, self.container, self.obj), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) resp.read() self.assertEqual(resp.status, 403) # create a shared container writable by account3 shared_container = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', '%s/%s' % ( parsed.path, shared_container), '', {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2], 'X-Container-Write': swift_test_perm[2]}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # verify third account can not copy from private container def copy(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( parsed.path, shared_container, 'private_object'), '', {'X-Auth-Token': token, 'Content-Length': '0', 'X-Copy-From': '%s/%s' % (self.container, self.obj)}) return check_response(conn) resp = retry(copy, use_account=3) resp.read() self.assertEqual(resp.status, 403) # verify third account can write "obj1" to shared container def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/%s' % ( parsed.path, shared_container, 'obj1'), 'test', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put, use_account=3) resp.read() self.assertEqual(resp.status, 201) # verify third account can copy "obj1" to shared container def copy2(url, token, parsed, conn): conn.request('COPY', '%s/%s/%s' % ( parsed.path, shared_container, 'obj1'), '', {'X-Auth-Token': token, 'Destination': '%s/%s' % (shared_container, 'obj1')}) return check_response(conn) resp = retry(copy2, use_account=3) resp.read() self.assertEqual(resp.status, 201) # verify third account STILL can not copy from private container def copy3(url, token, parsed, conn): conn.request('COPY', '%s/%s/%s' % ( parsed.path, self.container, self.obj), '', {'X-Auth-Token': token, 'Destination': '%s/%s' % (shared_container, 'private_object')}) return check_response(conn) resp = retry(copy3, use_account=3) resp.read() self.assertEqual(resp.status, 403) # clean up "obj1" def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/%s' % ( parsed.path, shared_container, 'obj1'), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) # clean up shared_container def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + shared_container, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) @requires_acls def test_read_only(self): if skip3: raise SkipTest def get_listing(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def get(url, token, parsed, conn, name): conn.request('GET', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, name), 'test', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) # cannot list objects resp = retry(get_listing, use_account=3) resp.read() self.assertEquals(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-only access acl_user = swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.obj in listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 200) self.assertEquals(body, 'test') # can not put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() self.assertEquals(resp.status, 403) # can not delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 403) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(obj_name not in listing) self.assert_(self.obj in listing) @requires_acls def test_read_write(self): if skip3: raise SkipTest def get_listing(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def get(url, token, parsed, conn, name): conn.request('GET', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, name), 'test', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) # cannot list objects resp = retry(get_listing, use_account=3) resp.read() self.assertEquals(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-write access acl_user = swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.obj in listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 200) self.assertEquals(body, 'test') # can put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() self.assertEquals(resp.status, 201) # can delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 204) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(obj_name in listing) self.assert_(self.obj not in listing) @requires_acls def test_admin(self): if skip3: raise SkipTest def get_listing(url, token, parsed, conn): conn.request('GET', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) def post_account(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def get(url, token, parsed, conn, name): conn.request('GET', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) def put(url, token, parsed, conn, name): conn.request('PUT', '%s/%s/%s' % ( parsed.path, self.container, name), 'test', {'X-Auth-Token': token}) return check_response(conn) def delete(url, token, parsed, conn, name): conn.request('DELETE', '%s/%s/%s' % ( parsed.path, self.container, name), '', {'X-Auth-Token': token}) return check_response(conn) # cannot list objects resp = retry(get_listing, use_account=3) resp.read() self.assertEquals(resp.status, 403) # cannot get object resp = retry(get, self.obj, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant admin access acl_user = swift_test_user[2] acl = {'admin': [acl_user]} headers = {'x-account-access-control': json.dumps(acl)} resp = retry(post_account, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # can list objects resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(self.obj in listing) # can get object resp = retry(get, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 200) self.assertEquals(body, 'test') # can put an object obj_name = str(uuid4()) resp = retry(put, obj_name, use_account=3) body = resp.read() self.assertEquals(resp.status, 201) # can delete an object resp = retry(delete, self.obj, use_account=3) body = resp.read() self.assertEquals(resp.status, 204) # sanity with account1 resp = retry(get_listing, use_account=3) listing = resp.read() self.assertEquals(resp.status, 200) self.assert_(obj_name in listing) self.assert_(self.obj not in listing) def test_manifest(self): if skip: raise SkipTest # Data for the object segments segments1 = ['one', 'two', 'three', 'four', 'five'] segments2 = ['six', 'seven', 'eight'] segments3 = ['nine', 'ten', 'eleven'] # Upload the first set of segments def put(url, token, parsed, conn, objnum): conn.request('PUT', '%s/%s/segments1/%s' % ( parsed.path, self.container, str(objnum)), segments1[objnum], {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments1)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) # Upload the manifest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/manifest' % ( parsed.path, self.container), '', { 'X-Auth-Token': token, 'X-Object-Manifest': '%s/segments1/' % self.container, 'Content-Type': 'text/jibberish', 'Content-Length': '0'}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # Get the manifest (should get all the segments as the body) def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments1)) self.assertEqual(resp.status, 200) self.assertEqual(resp.getheader('content-type'), 'text/jibberish') # Get with a range at the start of the second segment def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', { 'X-Auth-Token': token, 'Range': 'bytes=3-'}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments1[1:])) self.assertEqual(resp.status, 206) # Get with a range in the middle of the second segment def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', { 'X-Auth-Token': token, 'Range': 'bytes=5-'}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments1)[5:]) self.assertEqual(resp.status, 206) # Get with a full start and stop range def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', { 'X-Auth-Token': token, 'Range': 'bytes=5-10'}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments1)[5:11]) self.assertEqual(resp.status, 206) # Upload the second set of segments def put(url, token, parsed, conn, objnum): conn.request('PUT', '%s/%s/segments2/%s' % ( parsed.path, self.container, str(objnum)), segments2[objnum], {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments2)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) # Get the manifest (should still be the first segments of course) def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments1)) self.assertEqual(resp.status, 200) # Update the manifest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/manifest' % ( parsed.path, self.container), '', { 'X-Auth-Token': token, 'X-Object-Manifest': '%s/segments2/' % self.container, 'Content-Length': '0'}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # Get the manifest (should be the second set of segments now) def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments2)) self.assertEqual(resp.status, 200) if not skip3: # Ensure we can't access the manifest with the third account def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) resp.read() self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): conn.request('POST', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) self.assertEqual(resp.read(), ''.join(segments2)) self.assertEqual(resp.status, 200) # Create another container for the third set of segments acontainer = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', parsed.path + '/' + acontainer, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # Upload the third set of segments in the other container def put(url, token, parsed, conn, objnum): conn.request('PUT', '%s/%s/segments3/%s' % ( parsed.path, acontainer, str(objnum)), segments3[objnum], {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments3)): resp = retry(put, objnum) resp.read() self.assertEqual(resp.status, 201) # Update the manifest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token, 'X-Object-Manifest': '%s/segments3/' % acontainer, 'Content-Length': '0'}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) # Get the manifest to ensure it's the third set of segments def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get) self.assertEqual(resp.read(), ''.join(segments3)) self.assertEqual(resp.status, 200) if not skip3: # Ensure we can't access the manifest with the third account # (because the segments are in a protected container even if the # manifest itself is not). def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) resp.read() self.assertEqual(resp.status, 403) # Grant access to the third account def post(url, token, parsed, conn): conn.request('POST', '%s/%s' % (parsed.path, acontainer), '', {'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]}) return check_response(conn) resp = retry(post) resp.read() self.assertEqual(resp.status, 204) # The third account should be able to get the manifest now def get(url, token, parsed, conn): conn.request('GET', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(get, use_account=3) self.assertEqual(resp.read(), ''.join(segments3)) self.assertEqual(resp.status, 200) # Delete the manifest def delete(url, token, parsed, conn, objnum): conn.request('DELETE', '%s/%s/manifest' % ( parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) # Delete the third set of segments def delete(url, token, parsed, conn, objnum): conn.request('DELETE', '%s/%s/segments3/%s' % ( parsed.path, acontainer, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments3)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) # Delete the second set of segments def delete(url, token, parsed, conn, objnum): conn.request('DELETE', '%s/%s/segments2/%s' % ( parsed.path, self.container, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments2)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) # Delete the first set of segments def delete(url, token, parsed, conn, objnum): conn.request('DELETE', '%s/%s/segments1/%s' % ( parsed.path, self.container, str(objnum)), '', {'X-Auth-Token': token}) return check_response(conn) for objnum in xrange(len(segments1)): resp = retry(delete, objnum) resp.read() self.assertEqual(resp.status, 204) # Delete the extra container def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s' % (parsed.path, acontainer), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) def test_delete_content_type(self): if skip: raise SkipTest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/hi' % (parsed.path, self.container), 'there', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('Content-Type'), 'text/html; charset=UTF-8') def test_delete_if_delete_at_bad(self): if skip: raise SkipTest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/hi-delete-bad' % (parsed.path, self.container), 'there', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) resp.read() self.assertEqual(resp.status, 201) def delete(url, token, parsed, conn): conn.request('DELETE', '%s/%s/hi' % (parsed.path, self.container), '', {'X-Auth-Token': token, 'X-If-Delete-At': 'bad'}) return check_response(conn) resp = retry(delete) resp.read() self.assertEqual(resp.status, 400) def test_null_name(self): if skip: raise SkipTest def put(url, token, parsed, conn): conn.request('PUT', '%s/%s/abc%%00def' % ( parsed.path, self.container), 'test', {'X-Auth-Token': token}) return check_response(conn) resp = retry(put) if (web_front_end == 'apache2'): self.assertEqual(resp.status, 404) else: self.assertEqual(resp.read(), 'Invalid UTF8 or contains NULL') self.assertEqual(resp.status, 412) def test_cors(self): if skip: raise SkipTest def is_strict_mode(url, token, parsed, conn): conn.request('GET', '/info') resp = conn.getresponse() if resp.status // 100 == 2: info = json.loads(resp.read()) return info.get('swift', {}).get('strict_cors_mode', False) return False def put_cors_cont(url, token, parsed, conn, orig): conn.request( 'PUT', '%s/%s' % (parsed.path, self.container), '', {'X-Auth-Token': token, 'X-Container-Meta-Access-Control-Allow-Origin': orig}) return check_response(conn) def put_obj(url, token, parsed, conn, obj): conn.request( 'PUT', '%s/%s/%s' % (parsed.path, self.container, obj), 'test', {'X-Auth-Token': token}) return check_response(conn) def check_cors(url, token, parsed, conn, method, obj, headers): if method != 'OPTIONS': headers['X-Auth-Token'] = token conn.request( method, '%s/%s/%s' % (parsed.path, self.container, obj), '', headers) return conn.getresponse() strict_cors = retry(is_strict_mode) resp = retry(put_cors_cont, '*') resp.read() self.assertEquals(resp.status // 100, 2) resp = retry(put_obj, 'cat') resp.read() self.assertEquals(resp.status // 100, 2) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com'}) self.assertEquals(resp.status, 401) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com', 'Access-Control-Request-Method': 'GET'}) self.assertEquals(resp.status, 200) resp.read() headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertEquals(headers.get('access-control-allow-origin'), '*') resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) self.assertEquals(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertEquals(headers.get('access-control-allow-origin'), '*') resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com', 'X-Web-Mode': 'True'}) self.assertEquals(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertEquals(headers.get('access-control-allow-origin'), '*') #################### resp = retry(put_cors_cont, 'http://secret.com') resp.read() self.assertEquals(resp.status // 100, 2) resp = retry(check_cors, 'OPTIONS', 'cat', {'Origin': 'http://m.com', 'Access-Control-Request-Method': 'GET'}) resp.read() self.assertEquals(resp.status, 401) if strict_cors: resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) resp.read() self.assertEquals(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertTrue('access-control-allow-origin' not in headers) resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://secret.com'}) resp.read() self.assertEquals(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertEquals(headers.get('access-control-allow-origin'), 'http://secret.com') else: resp = retry(check_cors, 'GET', 'cat', {'Origin': 'http://m.com'}) resp.read() self.assertEquals(resp.status, 200) headers = dict((k.lower(), v) for k, v in resp.getheaders()) self.assertEquals(headers.get('access-control-allow-origin'), 'http://m.com') if __name__ == '__main__': unittest.main() swift-1.13.1/test/functional/test_account.py0000775000175400017540000007322112323703614022266 0ustar jenkinsjenkins00000000000000#!/usr/bin/python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import json from uuid import uuid4 from nose import SkipTest from string import letters from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift.common.middleware.acl import format_acl from swift_testing import (check_response, retry, skip, skip2, skip3, web_front_end, requires_acls) import swift_testing from test.functional.tests import load_constraint class TestAccount(unittest.TestCase): def test_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, value): conn.request('POST', parsed.path, '', {'X-Auth-Token': token, 'X-Account-Meta-Test': value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(post, '') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-test'), None) resp = retry(post, 'Value') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') resp = retry(get) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') def test_invalid_acls(self): def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # needs to be an acceptable header size num_keys = 8 max_key_size = load_constraint('max_header_size') / num_keys acl = {'admin': [c * max_key_size for c in letters[:num_keys]]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 400) # and again a touch smaller acl = {'admin': [c * max_key_size for c in letters[:num_keys - 1]]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) @requires_acls def test_invalid_acl_keys(self): def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # needs to be json resp = retry(post, headers={'X-Account-Access-Control': 'invalid'}, use_account=1) resp.read() self.assertEqual(resp.status, 400) acl_user = swift_testing.swift_test_user[1] acl = {'admin': [acl_user], 'invalid_key': 'invalid_value'} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers, use_account=1) resp.read() self.assertEqual(resp.status, 400) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) @requires_acls def test_invalid_acl_values(self): def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) acl = {'admin': 'invalid_value'} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 400) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) @requires_acls def test_read_only_acl(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # cannot read account resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read access acl_user = swift_testing.swift_test_user[2] acl = {'read-only': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-only can read account headers resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204)) # but not acls self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # read-only can not write metadata headers = {'x-account-meta-test': 'value'} resp = retry(post, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 403) # but they can read it headers = {'x-account-meta-test': 'value'} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204)) self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value') @requires_acls def test_read_write_acl(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # cannot read account resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant read-write access acl_user = swift_testing.swift_test_user[2] acl = {'read-write': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-write can read account headers resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204)) # but not acls self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # read-write can not write account metadata headers = {'x-account-meta-test': 'value'} resp = retry(post, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 403) @requires_acls def test_admin_acl(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # cannot read account resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) # grant admin access acl_user = swift_testing.swift_test_user[2] acl = {'admin': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # admin can read account headers resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204)) # including acls self.assertEqual(resp.getheader('X-Account-Access-Control'), acl_json_str) # admin can write account metadata value = str(uuid4()) headers = {'x-account-meta-test': value} resp = retry(post, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204)) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # admin can even revoke their own access headers = {'x-account-access-control': '{}'} resp = retry(post, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) # and again, cannot read account resp = retry(get, use_account=3) resp.read() self.assertEquals(resp.status, 403) @requires_acls def test_protected_tempurl(self): if skip3: raise SkipTest def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) # add a account metadata, and temp-url-key to account value = str(uuid4()) headers = { 'x-account-meta-temp-url-key': 'secret', 'x-account-meta-test': value, } resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # grant read-only access to tester3 acl_user = swift_testing.swift_test_user[2] acl = {'read-only': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-only tester3 can read account metadata resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204), 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # but not temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) # grant read-write access to tester3 acl_user = swift_testing.swift_test_user[2] acl = {'read-write': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # read-write tester3 can read account metadata resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204), 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # but not temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), None) # grant admin access to tester3 acl_user = swift_testing.swift_test_user[2] acl = {'admin': [acl_user]} acl_json_str = format_acl(version=2, acl_dict=acl) headers = {'x-account-access-control': acl_json_str} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # admin tester3 can read account metadata resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204), 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) # including temp-url-key self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), 'secret') # admin tester3 can even change temp-url-key secret = str(uuid4()) headers = { 'x-account-meta-temp-url-key': secret, } resp = retry(post, headers=headers, use_account=3) resp.read() self.assertEqual(resp.status, 204) resp = retry(get, use_account=3) resp.read() self.assert_(resp.status in (200, 204), 'Expected status in (200, 204), got %s' % resp.status) self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), secret) @requires_acls def test_account_acls(self): if skip2: raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('PUT', parsed.path, '', new_headers) return check_response(conn) def delete(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('DELETE', parsed.path, '', new_headers) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) try: # User1 can POST to their own account (and reset the ACLs) resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can GET their own empty account resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User2 can't GET User1's account resp = retry(get, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 403) # User1 is swift_owner of their own account, so they can POST an # ACL -- let's do this and make User2 (test_user[1]) an admin acl_user = swift_testing.swift_test_user[1] acl = {'admin': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User1 can see the new header resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) data_from_headers = resp.getheader('x-account-access-control') expected = json.dumps(acl, separators=(',', ':')) self.assertEqual(data_from_headers, expected) # Now User2 should be able to GET the account and see the ACL resp = retry(head, use_account=2, url_account=1) resp.read() data_from_headers = resp.getheader('x-account-access-control') self.assertEqual(data_from_headers, expected) # Revoke User2's admin access, grant User2 read-write access acl = {'read-write': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can PUT and DELETE a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 204) # Revoke User2's read-write access, grant User2 read-only access acl = {'read-only': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can't PUT a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 403) finally: # Make sure to clean up even if tests fail -- User2 should not # have access to User1's account in other functional tests! resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read() @requires_acls def test_swift_account_acls(self): if skip: raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) try: # User1 can POST to their own account resp = retry(post, headers={'X-Account-Access-Control': '{}'}) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can GET their own empty account resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can POST non-empty data acl_json = '{"admin":["bob"]}' resp = retry(post, headers={'X-Account-Access-Control': acl_json}) resp.read() self.assertEqual(resp.status, 204) # User1 can GET the non-empty data resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), acl_json) # POST non-JSON ACL should fail resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) resp.read() # resp.status will be 400 if tempauth or some other ACL-aware # auth middleware rejects it, or 200 (but silently swallowed by # core Swift) if ACL-unaware auth middleware approves it. # A subsequent GET should show the old, valid data, not the garbage resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), acl_json) finally: # Make sure to clean up even if tests fail -- User2 should not # have access to User1's account in other functional tests! resp = retry(post, headers={'X-Account-Access-Control': '{}'}) resp.read() def test_swift_prohibits_garbage_account_acls(self): if skip: raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) try: # User1 can POST to their own account resp = retry(post, headers={'X-Account-Access-Control': '{}'}) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can GET their own empty account resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can POST non-empty data acl_json = '{"admin":["bob"]}' resp = retry(post, headers={'X-Account-Access-Control': acl_json}) resp.read() self.assertEqual(resp.status, 204) # If this request is handled by ACL-aware auth middleware, then the # ACL will be persisted. If it is handled by ACL-unaware auth # middleware, then the header will be thrown out. But the request # should return successfully in any case. # User1 can GET the non-empty data resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) # ACL will be set if some ACL-aware auth middleware (e.g. tempauth) # propagates it to sysmeta; if no ACL-aware auth middleware does, # then X-Account-Access-Control will still be empty. # POST non-JSON ACL should fail resp = retry(post, headers={'X-Account-Access-Control': 'yuck'}) resp.read() # resp.status will be 400 if tempauth or some other ACL-aware # auth middleware rejects it, or 200 (but silently swallowed by # core Swift) if ACL-unaware auth middleware approves it. # A subsequent GET should either show the old, valid data (if # ACL-aware auth middleware is propagating it) or show nothing # (if no auth middleware in the pipeline is ACL-aware), but should # never return the garbage ACL. resp = retry(get) resp.read() self.assertEqual(resp.status // 100, 2) self.assertNotEqual(resp.getheader('X-Account-Access-Control'), 'yuck') finally: # Make sure to clean up even if tests fail -- User2 should not # have access to User1's account in other functional tests! resp = retry(post, headers={'X-Account-Access-Control': '{}'}) resp.read() def test_unicode_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path, '', {'X-Auth-Token': token, name: value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) uni_key = u'X-Account-Meta-uni\u0E12' uni_value = u'uni\u0E12' if (web_front_end == 'integral'): resp = retry(post, uni_key, '1') resp.read() self.assertTrue(resp.status in (201, 204)) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') resp = retry(post, 'X-Account-Meta-uni', uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('X-Account-Meta-uni'), uni_value.encode('utf-8')) if (web_front_end == 'integral'): resp = retry(post, uni_key, uni_value) resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader(uni_key.encode('utf-8')), uni_value.encode('utf-8')) def test_multi_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path, '', {'X-Auth-Token': token, name: value}) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) resp = retry(post, 'X-Account-Meta-One', '1') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-one'), '1') resp = retry(post, 'X-Account-Meta-Two', '2') resp.read() self.assertEqual(resp.status, 204) resp = retry(head) resp.read() self.assert_(resp.status in (200, 204), resp.status) self.assertEqual(resp.getheader('x-account-meta-one'), '1') self.assertEqual(resp.getheader('x-account-meta-two'), '2') def test_bad_metadata(self): if skip: raise SkipTest def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) conn.request('POST', parsed.path, '', headers) return check_response(conn) resp = retry(post, {'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'}) resp.read() self.assertEqual(resp.status, 400) resp = retry(post, {'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH}) resp.read() self.assertEqual(resp.status, 204) resp = retry( post, {'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)}) resp.read() self.assertEqual(resp.status, 400) headers = {} for x in xrange(MAX_META_COUNT): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers = {} for x in xrange(MAX_META_COUNT + 1): headers['X-Account-Meta-%d' % x] = 'v' resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) headers = {} header_value = 'k' * MAX_META_VALUE_LENGTH size = 0 x = 0 while size < MAX_META_OVERALL_SIZE - 4 - MAX_META_VALUE_LENGTH: size += 4 + MAX_META_VALUE_LENGTH headers['X-Account-Meta-%04d' % x] = header_value x += 1 if MAX_META_OVERALL_SIZE - size > 1: headers['X-Account-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size - 1) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 204) headers['X-Account-Meta-k'] = \ 'v' * (MAX_META_OVERALL_SIZE - size) resp = retry(post, headers) resp.read() self.assertEqual(resp.status, 400) if __name__ == '__main__': unittest.main() swift-1.13.1/test/functional/__init__.py0000664000175400017540000000000012323703614021310 0ustar jenkinsjenkins00000000000000swift-1.13.1/test/__init__.py0000664000175400017540000000467212323703611017166 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # See http://code.google.com/p/python-nose/issues/detail?id=373 # The code below enables nosetests to work with i18n _() blocks import sys import os try: from unittest.util import safe_repr except ImportError: # Probably py26 _MAX_LENGTH = 80 def safe_repr(obj, short=False): try: result = repr(obj) except Exception: result = object.__repr__(obj) if not short or len(result) < _MAX_LENGTH: return result return result[:_MAX_LENGTH] + ' [truncated]...' # make unittests pass on all locale import swift setattr(swift, 'gettext_', lambda x: x) from swift.common.utils import readconf # Work around what seems to be a Python bug. # c.f. https://bugs.launchpad.net/swift/+bug/820185. import logging logging.raiseExceptions = False def get_config(section_name=None, defaults=None): """ Attempt to get a test config dictionary. :param section_name: the section to read (all sections if not defined) :param defaults: an optional dictionary namespace of defaults """ config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', '/etc/swift/test.conf') config = {} if defaults is not None: config.update(defaults) try: config = readconf(config_file, section_name) except SystemExit: if not os.path.exists(config_file): print >>sys.stderr, \ 'Unable to read test config %s - file not found' \ % config_file elif not os.access(config_file, os.R_OK): print >>sys.stderr, \ 'Unable to read test config %s - permission denied' \ % config_file else: print >>sys.stderr, \ 'Unable to read test config %s - section %s not found' \ % (config_file, section_name) return config swift-1.13.1/swift/0000775000175400017540000000000012323703665015232 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/account/0000775000175400017540000000000012323703665016666 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/account/replicator.py0000664000175400017540000000152212323703611021373 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.account.backend import AccountBroker, DATADIR from swift.common import db_replicator class AccountReplicator(db_replicator.Replicator): server_type = 'account' brokerclass = AccountBroker datadir = DATADIR default_port = 6002 swift-1.13.1/swift/account/utils.py0000664000175400017540000000730012323703611020367 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import time from xml.sax import saxutils from swift.common.swob import HTTPOk, HTTPNoContent from swift.common.utils import json, normalize_timestamp class FakeAccountBroker(object): """ Quacks like an account broker, but doesn't actually do anything. Responds like an account broker would for a real, empty account with no metadata. """ def get_info(self): now = normalize_timestamp(time.time()) return {'container_count': 0, 'object_count': 0, 'bytes_used': 0, 'created_at': now, 'put_timestamp': now} def list_containers_iter(self, *_, **__): return [] @property def metadata(self): return {} def account_listing_response(account, req, response_content_type, broker=None, limit='', marker='', end_marker='', prefix='', delimiter=''): if broker is None: broker = FakeAccountBroker() info = broker.get_info() resp_headers = { 'X-Account-Container-Count': info['container_count'], 'X-Account-Object-Count': info['object_count'], 'X-Account-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp']} resp_headers.update((key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '') account_list = broker.list_containers_iter(limit, marker, end_marker, prefix, delimiter) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: data.append({'subdir': name}) else: data.append({'name': name, 'count': object_count, 'bytes': bytes_used}) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = ['', '' % saxutils.quoteattr(account)] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: output_list.append( '' % saxutils.quoteattr(name)) else: item = '%s%s' \ '%s' % \ (saxutils.escape(name), object_count, bytes_used) output_list.append(item) output_list.append('') account_list = '\n'.join(output_list) else: if not account_list: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret swift-1.13.1/swift/account/server.py0000664000175400017540000003460612323703614020551 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time import traceback from swift import gettext_ as _ from eventlet import Timeout import swift.common.db from swift.account.backend import AccountBroker, DATADIR from swift.account.utils import account_listing_response from swift.common.db import DatabaseConnectionError, DatabaseAlreadyExists from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, config_true_value, \ json, timing_stats, replication from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc from swift.common.swob import HTTPAccepted, HTTPBadRequest, \ HTTPCreated, HTTPForbidden, HTTPInternalServerError, \ HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPConflict, Request, \ HTTPInsufficientStorage, HTTPException from swift.common.request_helpers import is_sys_or_user_meta class AccountController(object): """WSGI controller for the account server.""" def __init__(self, conf, logger=None): self.logger = logger or get_logger(conf, log_route='account-server') self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.root = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) def _get_account_broker(self, drive, part, account, **kwargs): hsh = hash_path(account) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') kwargs.setdefault('account', account) kwargs.setdefault('logger', self.logger) return AccountBroker(db_path, **kwargs) def _deleted_response(self, broker, req, resp, body=''): # We are here since either the account does not exist or # it exists but marked for deletion. headers = {} # Try to check if account exists and is marked for deletion try: if broker.is_status_deleted(): # Account does exist and is marked for deletion headers = {'X-Account-Status': 'Deleted'} except DatabaseConnectionError: # Account does not exist! pass return resp(request=req, headers=headers, charset='utf-8', body=body) @public @timing_stats() def DELETE(self, req): """Handle HTTP DELETE request.""" drive, part, account = split_and_validate_path(req, 3) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) broker.delete_db(req.headers['x-timestamp']) return self._deleted_response(broker, req, HTTPNoContent) @public @timing_stats() def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container pending_timeout = None if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(normalize_timestamp( req.headers.get('x-timestamp') or time.time())) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used']) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account broker = self._get_account_broker(drive, part, account) timestamp = normalize_timestamp(req.headers['x-timestamp']) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req) @public @timing_stats() def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account = split_and_validate_path(req, 3) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) info = broker.get_info() headers = { 'X-Account-Container-Count': info['container_count'], 'X-Account-Object-Count': info['object_count'], 'X-Account-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp']} headers.update((key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '') headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8') @public @timing_stats() def GET(self, req): """Handle HTTP GET request.""" drive, part, account = split_and_validate_path(req, 3) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') limit = ACCOUNT_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > ACCOUNT_LISTING_LIMIT: return HTTPPreconditionFailed(request=req, body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter) @public @replication @timing_stats() def REPLICATE(self, req): """ Handle HTTP REPLICATE request. Handler for RPC calls for account replication. """ post_args = split_and_validate_path(req, 3) drive, partition, hash = post_args if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) try: args = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) ret.request = req return ret @public @timing_stats() def POST(self, req): """Handle HTTP POST request.""" drive, part, account = split_and_validate_path(req, 3) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req) def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] if self.log_requests: log_msg = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_msg) else: self.logger.info(log_msg) return res(env, start_response) def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI account server apps""" conf = global_conf.copy() conf.update(local_conf) return AccountController(conf) swift-1.13.1/swift/account/auditor.py0000664000175400017540000001233212323703611020677 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time from swift import gettext_ as _ from random import random import swift.common.db from swift.account.backend import AccountBroker, DATADIR from swift.common.utils import get_logger, audit_location_generator, \ config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon from eventlet import Timeout class AccountAuditor(Daemon): """Audit accounts.""" def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='account-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.interval = int(conf.get('interval', 1800)) self.account_passes = 0 self.account_failures = 0 self.accounts_running_time = 0 self.max_accounts_per_second = \ float(conf.get('accounts_per_second', 200)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "account.recon") def _one_audit_pass(self, reported): all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.account_audit(path) if time.time() - reported >= 3600: # once an hour self.logger.info(_('Since %(time)s: Account audits: ' '%(passed)s passed audit,' '%(failed)s failed audit'), {'time': time.ctime(reported), 'passed': self.account_passes, 'failed': self.account_failures}) dump_recon_cache({'account_audits_since': reported, 'account_audits_passed': self.account_passes, 'account_audits_failed': self.account_failures}, self.rcache, self.logger) reported = time.time() self.account_passes = 0 self.account_failures = 0 self.accounts_running_time = ratelimit_sleep( self.accounts_running_time, self.max_accounts_per_second) return reported def run_forever(self, *args, **kwargs): """Run the account audit until stopped.""" reported = time.time() time.sleep(random() * self.interval) while True: self.logger.info(_('Begin account audit pass.')) begin = time.time() try: reported = self._one_audit_pass(reported) except (Exception, Timeout): self.logger.increment('errors') self.logger.exception(_('ERROR auditing')) elapsed = time.time() - begin if elapsed < self.interval: time.sleep(self.interval - elapsed) self.logger.info( _('Account audit pass completed: %.02fs'), elapsed) dump_recon_cache({'account_auditor_pass_completed': elapsed}, self.rcache, self.logger) def run_once(self, *args, **kwargs): """Run the account audit once.""" self.logger.info(_('Begin account audit "once" mode')) begin = reported = time.time() self._one_audit_pass(reported) elapsed = time.time() - begin self.logger.info( _('Account audit "once" mode completed: %.02fs'), elapsed) dump_recon_cache({'account_auditor_pass_completed': elapsed}, self.rcache, self.logger) def account_audit(self, path): """ Audits the given account path :param path: the path to an account db """ start_time = time.time() try: broker = AccountBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.account_passes += 1 self.logger.debug(_('Audit passed for %s') % broker) except (Exception, Timeout): self.logger.increment('failures') self.account_failures += 1 self.logger.exception(_('ERROR Could not get account info %s'), path) self.logger.timing_since('timing', start_time) swift-1.13.1/swift/account/reaper.py0000664000175400017540000005153012323703611020511 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import random from swift import gettext_ as _ from logging import DEBUG from math import sqrt from time import time, ctime from eventlet import GreenPool, sleep, Timeout import swift.common.db from swift.account.backend import AccountBroker, DATADIR from swift.common.direct_client import direct_delete_container, \ direct_delete_object, direct_get_container from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import get_logger, whataremyips, ismount, \ config_true_value from swift.common.daemon import Daemon class AccountReaper(Daemon): """ Removes data from status=DELETED accounts. These are accounts that have been asked to be removed by the reseller via services remove_storage_account XMLRPC call. The account is not deleted immediately by the services call, but instead the account is simply marked for deletion by setting the status column in the account_stat table of the account database. This account reaper scans for such accounts and removes the data in the background. The background deletion process will occur on the primary account server for the account. :param server_conf: The [account-server] dictionary of the account server configuration file :param reaper_conf: The [account-reaper] dictionary of the account server configuration file See the etc/account-server.conf-sample for information on the possible configuration parameters. """ def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='account-reaper') self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.interval = int(conf.get('interval', 3600)) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring = None self.container_ring = None self.object_ring = None self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.myips = whataremyips() self.concurrency = int(conf.get('concurrency', 25)) self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.delay_reaping = int(conf.get('delay_reaping') or 0) reap_warn_after = float(conf.get('reap_warn_after') or 86400 * 30) self.reap_not_done_after = reap_warn_after + self.delay_reaping def get_account_ring(self): """The account :class:`swift.common.ring.Ring` for the cluster.""" if not self.account_ring: self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring def get_container_ring(self): """The container :class:`swift.common.ring.Ring` for the cluster.""" if not self.container_ring: self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring def get_object_ring(self): """The object :class:`swift.common.ring.Ring` for the cluster.""" if not self.object_ring: self.object_ring = Ring(self.swift_dir, ring_name='object') return self.object_ring def run_forever(self, *args, **kwargs): """Main entry point when running the reaper in normal daemon mode. This repeatedly calls :func:`reap_once` no quicker than the configuration interval. """ self.logger.debug(_('Daemon started.')) sleep(random.random() * self.interval) while True: begin = time() self.run_once() elapsed = time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Main entry point when running the reaper in 'once' mode, where it will do a single pass over all accounts on the server. This is called repeatedly by :func:`run_forever`. This will call :func:`reap_device` once for each device on the server. """ self.logger.debug(_('Begin devices pass: %s'), self.devices) begin = time() try: for device in os.listdir(self.devices): if self.mount_check and not ismount( os.path.join(self.devices, device)): self.logger.increment('errors') self.logger.debug( _('Skipping %s as it is not mounted'), device) continue self.reap_device(device) except (Exception, Timeout): self.logger.exception(_("Exception in top-level account reaper " "loop")) elapsed = time() - begin self.logger.info(_('Devices pass completed: %.02fs'), elapsed) def reap_device(self, device): """ Called once per pass for each device on the server. This will scan the accounts directory for the device, looking for partitions this device is the primary for, then looking for account databases that are marked status=DELETED and still have containers and calling :func:`reap_account`. Account databases marked status=DELETED that no longer have containers will eventually be permanently removed by the reclaim process within the account replicator (see :mod:`swift.db_replicator`). :param device: The device to look for accounts to be deleted. """ datadir = os.path.join(self.devices, device, DATADIR) if not os.path.exists(datadir): return for partition in os.listdir(datadir): partition_path = os.path.join(datadir, partition) if not partition.isdigit(): continue nodes = self.get_account_ring().get_part_nodes(int(partition)) if nodes[0]['ip'] not in self.myips or \ not os.path.isdir(partition_path): continue for suffix in os.listdir(partition_path): suffix_path = os.path.join(partition_path, suffix) if not os.path.isdir(suffix_path): continue for hsh in os.listdir(suffix_path): hsh_path = os.path.join(suffix_path, hsh) if not os.path.isdir(hsh_path): continue for fname in sorted(os.listdir(hsh_path), reverse=True): if fname.endswith('.ts'): break elif fname.endswith('.db'): self.start_time = time() broker = \ AccountBroker(os.path.join(hsh_path, fname)) if broker.is_status_deleted() and \ not broker.empty(): self.reap_account(broker, partition, nodes) def reap_account(self, broker, partition, nodes): """ Called once per pass for each account this server is the primary for and attempts to delete the data for the given account. The reaper will only delete one account at any given time. It will call :func:`reap_container` up to sqrt(self.concurrency) times concurrently while reaping the account. If there is any exception while deleting a single container, the process will continue for any other containers and the failed containers will be tried again the next time this function is called with the same parameters. If there is any exception while listing the containers for deletion, the process will stop (but will obviously be tried again the next time this function is called with the same parameters). This isn't likely since the listing comes from the local database. After the process completes (successfully or not) statistics about what was accomplished will be logged. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param broker: The AccountBroker for the account to delete. :param partition: The partition in the account ring the account is on. :param nodes: The primary node dicts for the account to delete. .. seealso:: :class:`swift.account.backend.AccountBroker` for the broker class. .. seealso:: :func:`swift.common.ring.Ring.get_nodes` for a description of the node dicts. """ begin = time() info = broker.get_info() if time() - float(info['delete_timestamp']) <= self.delay_reaping: return False account = info['account'] self.logger.info(_('Beginning pass on account %s'), account) self.stats_return_codes = {} self.stats_containers_deleted = 0 self.stats_objects_deleted = 0 self.stats_containers_remaining = 0 self.stats_objects_remaining = 0 self.stats_containers_possibly_remaining = 0 self.stats_objects_possibly_remaining = 0 try: marker = '' while True: containers = \ list(broker.list_containers_iter(1000, marker, None, None, None)) if not containers: break try: for (container, _junk, _junk, _junk) in containers: self.container_pool.spawn(self.reap_container, account, partition, nodes, container) self.container_pool.waitall() except (Exception, Timeout): self.logger.exception( _('Exception with containers for account %s'), account) marker = containers[-1][0] if marker == '': break log = 'Completed pass on account %s' % account except (Exception, Timeout): self.logger.exception( _('Exception with account %s'), account) log = _('Incomplete pass on account %s') % account if self.stats_containers_deleted: log += _(', %s containers deleted') % self.stats_containers_deleted if self.stats_objects_deleted: log += _(', %s objects deleted') % self.stats_objects_deleted if self.stats_containers_remaining: log += _(', %s containers remaining') % \ self.stats_containers_remaining if self.stats_objects_remaining: log += _(', %s objects remaining') % self.stats_objects_remaining if self.stats_containers_possibly_remaining: log += _(', %s containers possibly remaining') % \ self.stats_containers_possibly_remaining if self.stats_objects_possibly_remaining: log += _(', %s objects possibly remaining') % \ self.stats_objects_possibly_remaining if self.stats_return_codes: log += _(', return codes: ') for code in sorted(self.stats_return_codes): log += '%s %sxxs, ' % (self.stats_return_codes[code], code) log = log[:-2] log += _(', elapsed: %.02fs') % (time() - begin) self.logger.info(log) self.logger.timing_since('timing', self.start_time) if self.stats_containers_remaining and \ begin - float(info['delete_timestamp']) >= self.reap_not_done_after: self.logger.warn(_('Account %s has not been reaped since %s') % (account, ctime(float(info['delete_timestamp'])))) return True def reap_container(self, account, account_partition, account_nodes, container): """ Deletes the data and the container itself for the given container. This will call :func:`reap_object` up to sqrt(self.concurrency) times concurrently for the objects in the container. If there is any exception while deleting a single object, the process will continue for any other objects in the container and the failed objects will be tried again the next time this function is called with the same parameters. If there is any exception while listing the objects for deletion, the process will stop (but will obviously be tried again the next time this function is called with the same parameters). This is a possibility since the listing comes from querying just the primary remote container server. Once all objects have been attempted to be deleted, the container itself will be attempted to be deleted by sending a delete request to all container nodes. The format of the delete request is such that each container server will update a corresponding account server, removing the container from the account's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the container. :param account_partition: The partition for the account on the account ring. :param account_nodes: The primary node dicts for the account. :param container: The name of the container to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the account node dicts. """ account_nodes = list(account_nodes) part, nodes = self.get_container_ring().get_nodes(account, container) node = nodes[-1] pool = GreenPool(size=self.object_concurrency) marker = '' while True: objects = None try: objects = direct_get_container( node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout)[1] self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if not objects: break try: for obj in objects: if isinstance(obj['name'], unicode): obj['name'] = obj['name'].encode('utf8') pool.spawn(self.reap_object, account, container, part, nodes, obj['name']) pool.waitall() except (Exception, Timeout): self.logger.exception(_('Exception with objects for container ' '%(container)s for account %(account)s' ), {'container': container, 'account': account}) marker = objects[-1]['name'] if marker == '': break successes = 0 failures = 0 for node in nodes: anode = account_nodes.pop() try: direct_delete_container( node, part, account, container, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={'X-Account-Host': '%(ip)s:%(port)s' % anode, 'X-Account-Partition': str(account_partition), 'X-Account-Device': anode['device'], 'X-Account-Override-Deleted': 'yes'}) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('containers_failures') self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if successes > failures: self.stats_containers_deleted += 1 self.logger.increment('containers_deleted') elif not successes: self.stats_containers_remaining += 1 self.logger.increment('containers_remaining') else: self.stats_containers_possibly_remaining += 1 self.logger.increment('containers_possibly_remaining') def reap_object(self, account, container, container_partition, container_nodes, obj): """ Deletes the given object by issuing a delete request to each node for the object. The format of the delete request is such that each object server will update a corresponding container server, removing the object from the container's listing. This function returns nothing and should raise no exception but only update various self.stats_* values for what occurs. :param account: The name of the account for the object. :param container: The name of the container for the object. :param container_partition: The partition for the container on the container ring. :param container_nodes: The primary node dicts for the container. :param obj: The name of the object to delete. * See also: :func:`swift.common.ring.Ring.get_nodes` for a description of the container node dicts. """ container_nodes = list(container_nodes) part, nodes = self.get_object_ring().get_nodes(account, container, obj) successes = 0 failures = 0 for node in nodes: cnode = container_nodes.pop() try: direct_delete_object( node, part, account, container, obj, conn_timeout=self.conn_timeout, response_timeout=self.node_timeout, headers={'X-Container-Host': '%(ip)s:%(port)s' % cnode, 'X-Container-Partition': str(container_partition), 'X-Container-Device': cnode['device']}) successes += 1 self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 self.logger.increment('return_codes.2') except ClientException as err: if self.logger.getEffectiveLevel() <= DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) failures += 1 self.logger.increment('objects_failures') self.stats_return_codes[err.http_status / 100] = \ self.stats_return_codes.get(err.http_status / 100, 0) + 1 self.logger.increment( 'return_codes.%d' % (err.http_status / 100,)) if successes > failures: self.stats_objects_deleted += 1 self.logger.increment('objects_deleted') elif not successes: self.stats_objects_remaining += 1 self.logger.increment('objects_remaining') else: self.stats_objects_possibly_remaining += 1 self.logger.increment('objects_possibly_remaining') swift-1.13.1/swift/account/backend.py0000664000175400017540000004046412323703611020626 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Pluggable Back-end for Account Server """ import os from uuid import uuid4 import time import cPickle as pickle import errno import sqlite3 from swift.common.utils import normalize_timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode DATADIR = 'accounts' class AccountBroker(DatabaseBroker): """Encapsulates working with an account database.""" db_type = 'account' db_contains_type = 'container' db_reclaim_timestamp = 'delete_timestamp' def _initialize(self, conn, put_timestamp): """ Create a brand new account database (tables, indices, triggers, etc.) :param conn: DB connection object :param put_timestamp: put timestamp """ if not self.account: raise ValueError( 'Attempting to create a new database with no account set') self.create_container_table(conn) self.create_account_stat_table(conn, put_timestamp) def create_container_table(self, conn): """ Create container table which is specific to the account DB. :param conn: DB connection object """ conn.executescript(""" CREATE TABLE container ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, put_timestamp TEXT, delete_timestamp TEXT, object_count INTEGER, bytes_used INTEGER, deleted INTEGER DEFAULT 0 ); CREATE INDEX ix_container_deleted_name ON container (deleted, name); CREATE TRIGGER container_insert AFTER INSERT ON container BEGIN UPDATE account_stat SET container_count = container_count + (1 - new.deleted), object_count = object_count + new.object_count, bytes_used = bytes_used + new.bytes_used, hash = chexor(hash, new.name, new.put_timestamp || '-' || new.delete_timestamp || '-' || new.object_count || '-' || new.bytes_used); END; CREATE TRIGGER container_update BEFORE UPDATE ON container BEGIN SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); END; CREATE TRIGGER container_delete AFTER DELETE ON container BEGIN UPDATE account_stat SET container_count = container_count - (1 - old.deleted), object_count = object_count - old.object_count, bytes_used = bytes_used - old.bytes_used, hash = chexor(hash, old.name, old.put_timestamp || '-' || old.delete_timestamp || '-' || old.object_count || '-' || old.bytes_used); END; """) def create_account_stat_table(self, conn, put_timestamp): """ Create account_stat table which is specific to the account DB. Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object :param put_timestamp: put timestamp """ conn.executescript(""" CREATE TABLE account_stat ( account TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', container_count INTEGER, object_count INTEGER DEFAULT 0, bytes_used INTEGER DEFAULT 0, hash TEXT default '00000000000000000000000000000000', id TEXT, status TEXT DEFAULT '', status_changed_at TEXT DEFAULT '0', metadata TEXT DEFAULT '' ); INSERT INTO account_stat (container_count) VALUES (0); """) conn.execute(''' UPDATE account_stat SET account = ?, created_at = ?, id = ?, put_timestamp = ? ''', (self.account, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) def get_db_version(self, conn): if self._db_version == -1: self._db_version = 0 for row in conn.execute(''' SELECT name FROM sqlite_master WHERE name = 'ix_container_deleted_name' '''): self._db_version = 1 return self._db_version def _delete_db(self, conn, timestamp, force=False): """ Mark the DB as deleted. :param conn: DB connection object :param timestamp: timestamp to mark as deleted """ conn.execute(""" UPDATE account_stat SET delete_timestamp = ?, status = 'DELETED', status_changed_at = ? WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) def _commit_puts_load(self, item_list, entry): """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) = \ pickle.loads(entry.decode('base64')) item_list.append( {'name': name, 'put_timestamp': put_timestamp, 'delete_timestamp': delete_timestamp, 'object_count': object_count, 'bytes_used': bytes_used, 'deleted': deleted}) def empty(self): """ Check if the account DB is empty. :returns: True if the database has no active containers. """ self._commit_puts_stale_ok() with self.get() as conn: row = conn.execute( 'SELECT container_count from account_stat').fetchone() return (row[0] == 0) def put_container(self, name, put_timestamp, delete_timestamp, object_count, bytes_used): """ Create a container with the given attributes. :param name: name of the container to create :param put_timestamp: put_timestamp of the container to create :param delete_timestamp: delete_timestamp of the container to create :param object_count: number of objects in the container :param bytes_used: number of bytes used by the container """ if delete_timestamp > put_timestamp and \ object_count in (None, '', 0, '0'): deleted = 1 else: deleted = 0 record = {'name': name, 'put_timestamp': put_timestamp, 'delete_timestamp': delete_timestamp, 'object_count': object_count, 'bytes_used': bytes_used, 'deleted': deleted} if self.db_file == ':memory:': self.merge_items([record]) return if not os.path.exists(self.db_file): raise DatabaseConnectionError(self.db_file, "DB doesn't exist") pending_size = 0 try: pending_size = os.path.getsize(self.pending_file) except OSError as err: if err.errno != errno.ENOENT: raise if pending_size > PENDING_CAP: self._commit_puts([record]) else: with lock_parent_directory(self.pending_file, self.pending_timeout): with open(self.pending_file, 'a+b') as fp: # Colons aren't used in base64 encoding; so they are our # delimiter fp.write(':') fp.write(pickle.dumps( (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted), protocol=PICKLE_PROTOCOL).encode('base64')) fp.flush() def is_deleted(self): """ Check if the account DB is considered to be deleted. :returns: True if the account DB is considered to be deleted, False otherwise """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True self._commit_puts_stale_ok() with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, container_count, status FROM account_stat''').fetchone() return row['status'] == 'DELETED' or ( row['container_count'] in (None, '', 0, '0') and row['delete_timestamp'] > row['put_timestamp']) def is_status_deleted(self): """Only returns true if the status field is set to DELETED.""" with self.get() as conn: row = conn.execute(''' SELECT status FROM account_stat''').fetchone() return (row['status'] == "DELETED") def get_info(self): """ Get global data for the account. :returns: dict with keys: account, created_at, put_timestamp, delete_timestamp, container_count, object_count, bytes_used, hash, id """ self._commit_puts_stale_ok() with self.get() as conn: return dict(conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, container_count, object_count, bytes_used, hash, id FROM account_stat ''').fetchone()) def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :returns: list of tuples of (name, object_count, bytes_used, 0) """ (marker, end_marker, prefix, delimiter) = utf8encode( marker, end_marker, prefix, delimiter) self._commit_puts_stale_ok() if delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = """ SELECT name, object_count, bytes_used, 0 FROM container WHERE deleted = 0 AND """ query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: marker = name[:end] + chr(ord(delimiter) + 1) dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, 1]) curs.close() break results.append(row) if not rowcount: break return results def merge_items(self, item_list, source=None): """ Merge items into the container table. :param item_list: list of dictionaries of {'name', 'put_timestamp', 'delete_timestamp', 'object_count', 'bytes_used', 'deleted'} :param source: if defined, update incoming_sync with the source """ with self.get() as conn: max_rowid = -1 for rec in item_list: record = [rec['name'], rec['put_timestamp'], rec['delete_timestamp'], rec['object_count'], rec['bytes_used'], rec['deleted']] query = ''' SELECT name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted FROM container WHERE name = ? ''' if self.get_db_version(conn) >= 1: query += ' AND deleted IN (0, 1)' curs = conn.execute(query, (rec['name'],)) curs.row_factory = None row = curs.fetchone() if row: row = list(row) for i in xrange(5): if record[i] is None and row[i] is not None: record[i] = row[i] if row[1] > record[1]: # Keep newest put_timestamp record[1] = row[1] if row[2] > record[2]: # Keep newest delete_timestamp record[2] = row[2] # If deleted, mark as such if record[2] > record[1] and \ record[3] in (None, '', 0, '0'): record[5] = 1 else: record[5] = 0 conn.execute(''' DELETE FROM container WHERE name = ? AND deleted IN (0, 1) ''', (record[0],)) conn.execute(''' INSERT INTO container (name, put_timestamp, delete_timestamp, object_count, bytes_used, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', record) if source: max_rowid = max(max_rowid, rec['ROWID']) if source: try: conn.execute(''' INSERT INTO incoming_sync (sync_point, remote_id) VALUES (?, ?) ''', (max_rowid, source)) except sqlite3.IntegrityError: conn.execute(''' UPDATE incoming_sync SET sync_point=max(?, sync_point) WHERE remote_id=? ''', (max_rowid, source)) conn.commit() swift-1.13.1/swift/account/__init__.py0000664000175400017540000000000012323703611020754 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/common/0000775000175400017540000000000012323703665016522 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/common/bufferedhttp.py0000664000175400017540000001575212323703611021557 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Monkey Patch httplib.HTTPResponse to buffer reads of headers. This can improve performance when making large numbers of small HTTP requests. This module also provides helper functions to make HTTP connections using BufferedHTTPResponse. .. warning:: If you use this, be sure that the libraries you are using do not access the socket directly (xmlrpclib, I'm looking at you :/), and instead make all calls through httplib. """ from swift import gettext_ as _ from urllib import quote import logging import time from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \ HTTPResponse, HTTPSConnection, _UNKNOWN class BufferedHTTPResponse(HTTPResponse): """HTTPResponse class that buffers reading of headers""" def __init__(self, sock, debuglevel=0, strict=0, method=None): # pragma: no cover self.sock = sock # sock is an eventlet.greenio.GreenSocket # sock.fd is a socket._socketobject # sock.fd._sock is a socket._socket object, which is what we want. self._real_socket = sock.fd._sock self.fp = sock.makefile('rb') self.debuglevel = debuglevel self.strict = strict self._method = method self.msg = None # from the Status-Line of the response self.version = _UNKNOWN # HTTP-Version self.status = _UNKNOWN # Status-Code self.reason = _UNKNOWN # Reason-Phrase self.chunked = _UNKNOWN # is "chunked" being used? self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.length = _UNKNOWN # number of bytes left in response self.will_close = _UNKNOWN # conn will close at end of response def expect_response(self): if self.fp: self.fp.close() self.fp = None self.fp = self.sock.makefile('rb', 0) version, status, reason = self._read_status() if status != CONTINUE: self._read_status = lambda: (version, status, reason) self.begin() else: self.status = status self.reason = reason.strip() self.version = 11 self.msg = HTTPMessage(self.fp, 0) self.msg.fp = None def nuke_from_orbit(self): """ Terminate the socket with extreme prejudice. Closes the underlying socket regardless of whether or not anyone else has references to it. Use this when you are certain that nobody else you care about has a reference to this socket. """ if self._real_socket: # this is idempotent; see sock_close in Modules/socketmodule.c in # the Python source for details. self._real_socket.close() self._real_socket = None self.close() def close(self): HTTPResponse.close(self) self.sock = None self._real_socket = None class BufferedHTTPConnection(HTTPConnection): """HTTPConnection class that uses BufferedHTTPResponse""" response_class = BufferedHTTPResponse def connect(self): self._connected_time = time.time() return HTTPConnection.connect(self) def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): self._method = method self._path = url return HTTPConnection.putrequest(self, method, url, skip_host, skip_accept_encoding) def getexpect(self): response = BufferedHTTPResponse(self.sock, strict=self.strict, method=self._method) response.expect_response() return response def getresponse(self): response = HTTPConnection.getresponse(self) logging.debug(_("HTTP PERF: %(time).5f seconds to %(method)s " "%(host)s:%(port)s %(path)s)"), {'time': time.time() - self._connected_time, 'method': self._method, 'host': self.host, 'port': self.port, 'path': self._path}) return response def http_connect(ipaddr, port, device, partition, method, path, headers=None, query_string=None, ssl=False): """ Helper function to create an HTTPConnection object. If ssl is set True, HTTPSConnection will be used. However, if ssl=False, BufferedHTTPConnection will be used, which is buffered for backend Swift services. :param ipaddr: IPv4 address to connect to :param port: port to connect to :param device: device of the node to query :param partition: partition on the device :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) :param path: request path :param headers: dictionary of headers :param query_string: request query string :param ssl: set True if SSL should be used (default: False) :returns: HTTPConnection object """ if isinstance(path, unicode): try: path = path.encode("utf-8") except UnicodeError as e: logging.exception(_('Error encoding to UTF-8: %s'), str(e)) path = quote('/' + device + '/' + str(partition) + path) return http_connect_raw( ipaddr, port, method, path, headers, query_string, ssl) def http_connect_raw(ipaddr, port, method, path, headers=None, query_string=None, ssl=False): """ Helper function to create an HTTPConnection object. If ssl is set True, HTTPSConnection will be used. However, if ssl=False, BufferedHTTPConnection will be used, which is buffered for backend Swift services. :param ipaddr: IPv4 address to connect to :param port: port to connect to :param method: HTTP method to request ('GET', 'PUT', 'POST', etc.) :param path: request path :param headers: dictionary of headers :param query_string: request query string :param ssl: set True if SSL should be used (default: False) :returns: HTTPConnection object """ if not port: port = 443 if ssl else 80 if ssl: conn = HTTPSConnection('%s:%s' % (ipaddr, port)) else: conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) if query_string: path += '?' + query_string conn.path = path conn.putrequest(method, path, skip_host=(headers and 'Host' in headers)) if headers: for header, value in headers.iteritems(): conn.putheader(header, str(value)) conn.endheaders() return conn swift-1.13.1/swift/common/internal_client.py0000664000175400017540000007434712323703614022257 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from eventlet import sleep, Timeout from eventlet.green import httplib, socket, urllib2 import json import struct from sys import exc_info import zlib from swift import gettext_ as _ import urlparse from zlib import compressobj from swift.common.utils import quote from swift.common.http import HTTP_NOT_FOUND from swift.common.swob import Request from swift.common.wsgi import loadapp class UnexpectedResponse(Exception): """ Exception raised on invalid responses to InternalClient.make_request(). :param message: Exception message. :param resp: The unexpected response. """ def __init__(self, message, resp): super(UnexpectedResponse, self).__init__(message) self.resp = resp class CompressingFileReader(object): """ Wrapper for file object to compress object while reading. Can be used to wrap file objects passed to InternalClient.upload_object(). Used in testing of InternalClient. :param file_obj: File object to wrap. :param compresslevel: Compression level, defaults to 9. :param chunk_size: Size of chunks read when iterating using object, defaults to 4096. """ def __init__(self, file_obj, compresslevel=9, chunk_size=4096): self._f = file_obj self.compresslevel = compresslevel self.chunk_size = chunk_size self.set_initial_state() def set_initial_state(self): """ Sets the object to the state needed for the first read. """ self._f.seek(0) self._compressor = compressobj( self.compresslevel, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) self.done = False self.first = True self.crc32 = 0 self.total_size = 0 def read(self, *a, **kw): """ Reads a chunk from the file object. Params are passed directly to the underlying file object's read(). :returns: Compressed chunk from file object. """ if self.done: return '' x = self._f.read(*a, **kw) if x: self.crc32 = zlib.crc32(x, self.crc32) & 0xffffffffL self.total_size += len(x) compressed = self._compressor.compress(x) if not compressed: compressed = self._compressor.flush(zlib.Z_SYNC_FLUSH) else: compressed = self._compressor.flush(zlib.Z_FINISH) crc32 = struct.pack(" self.retries: raise sleep(backoff) backoff = min(backoff * 2, self.max_backoff) def get_account(self, *args, **kwargs): # Used in swift-dispersion-populate return self.retry_request('GET', **kwargs) def put_container(self, container, **kwargs): # Used in swift-dispersion-populate return self.retry_request('PUT', container=container, **kwargs) def get_container(self, container, **kwargs): # Used in swift-dispersion-populate return self.retry_request('GET', container=container, **kwargs) def put_object(self, container, name, contents, **kwargs): # Used in swift-dispersion-populate return self.retry_request('PUT', container=container, name=name, contents=contents.read(), **kwargs) def put_object(url, **kwargs): """For usage with container sync """ client = SimpleClient(url=url) client.retry_request('PUT', **kwargs) def delete_object(url, **kwargs): """For usage with container sync """ client = SimpleClient(url=url) client.retry_request('DELETE', **kwargs) swift-1.13.1/swift/common/manager.py0000664000175400017540000005377712323703611020520 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import functools import errno import os import resource import signal import time import subprocess import re from swift import gettext_ as _ from swift.common.utils import search_tree, remove_file, write_file SWIFT_DIR = '/etc/swift' RUN_DIR = '/var/run/swift' # auth-server has been removed from ALL_SERVERS, start it explicitly ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor', 'container-replicator', 'container-server', 'container-sync', 'container-updater', 'object-auditor', 'object-server', 'object-expirer', 'object-replicator', 'object-updater', 'proxy-server', 'account-replicator', 'account-reaper'] MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server', 'object-server'] REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS] GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server'] START_ONCE_SERVERS = REST_SERVERS # These are servers that match a type (account-*, container-*, object-*) but # don't use that type-server.conf file and instead use their own. STANDALONE_SERVERS = ['object-expirer'] KILL_WAIT = 15 # seconds to wait for servers to die (by default) WARNING_WAIT = 3 # seconds to wait after message that may just be a warning MAX_DESCRIPTORS = 32768 MAX_MEMORY = (1024 * 1024 * 1024) * 2 # 2 GB MAX_PROCS = 8192 # workers * disks * threads_per_disk, can get high def setup_env(): """Try to increase resource limits of the OS. Move PYTHON_EGG_CACHE to /tmp """ try: resource.setrlimit(resource.RLIMIT_NOFILE, (MAX_DESCRIPTORS, MAX_DESCRIPTORS)) except ValueError: print _("WARNING: Unable to modify file descriptor limit. " "Running as non-root?") try: resource.setrlimit(resource.RLIMIT_DATA, (MAX_MEMORY, MAX_MEMORY)) except ValueError: print _("WARNING: Unable to modify memory limit. " "Running as non-root?") try: resource.setrlimit(resource.RLIMIT_NPROC, (MAX_PROCS, MAX_PROCS)) except ValueError: print _("WARNING: Unable to modify max process limit. " "Running as non-root?") # Set PYTHON_EGG_CACHE if it isn't already set os.environ.setdefault('PYTHON_EGG_CACHE', '/tmp') def command(func): """ Decorator to declare which methods are accessible as commands, commands always return 1 or 0, where 0 should indicate success. :param func: function to make public """ func.publicly_accessible = True @functools.wraps(func) def wrapped(*a, **kw): rv = func(*a, **kw) return 1 if rv else 0 return wrapped def watch_server_pids(server_pids, interval=1, **kwargs): """Monitor a collection of server pids yielding back those pids that aren't responding to signals. :param server_pids: a dict, lists of pids [int,...] keyed on Server objects """ status = {} start = time.time() end = start + interval server_pids = dict(server_pids) # make a copy while True: for server, pids in server_pids.items(): for pid in pids: try: # let pid stop if it wants to os.waitpid(pid, os.WNOHANG) except OSError as e: if e.errno not in (errno.ECHILD, errno.ESRCH): raise # else no such child/process # check running pids for server status[server] = server.get_running_pids(**kwargs) for pid in pids: # original pids no longer in running pids! if pid not in status[server]: yield server, pid # update active pids list using running_pids server_pids[server] = status[server] if not [p for server, pids in status.items() for p in pids]: # no more running pids break if time.time() > end: break else: time.sleep(0.1) class UnknownCommandError(Exception): pass class Manager(object): """Main class for performing commands on groups of servers. :param servers: list of server names as strings """ def __init__(self, servers, run_dir=RUN_DIR): server_names = set() for server in servers: if server == 'all': server_names.update(ALL_SERVERS) elif server == 'main': server_names.update(MAIN_SERVERS) elif server == 'rest': server_names.update(REST_SERVERS) elif '*' in server: # convert glob to regex server_names.update([s for s in ALL_SERVERS if re.match(server.replace('*', '.*'), s)]) else: server_names.add(server) self.servers = set() for name in server_names: self.servers.add(Server(name, run_dir)) def __iter__(self): return iter(self.servers) @command def status(self, **kwargs): """display status of tracked pids for server """ status = 0 for server in self.servers: status += server.status(**kwargs) return status @command def start(self, **kwargs): """starts a server """ setup_env() status = 0 for server in self.servers: server.launch(**kwargs) if not kwargs.get('daemon', True): for server in self.servers: try: status += server.interact(**kwargs) except KeyboardInterrupt: print _('\nuser quit') self.stop(**kwargs) break elif kwargs.get('wait', True): for server in self.servers: status += server.wait(**kwargs) return status @command def no_wait(self, **kwargs): """spawn server and return immediately """ kwargs['wait'] = False return self.start(**kwargs) @command def no_daemon(self, **kwargs): """start a server interactively """ kwargs['daemon'] = False return self.start(**kwargs) @command def once(self, **kwargs): """start server and run one pass on supporting daemons """ kwargs['once'] = True return self.start(**kwargs) @command def stop(self, **kwargs): """stops a server """ server_pids = {} for server in self.servers: signaled_pids = server.stop(**kwargs) if not signaled_pids: print _('No %s running') % server else: server_pids[server] = signaled_pids # all signaled_pids, i.e. list(itertools.chain(*server_pids.values())) signaled_pids = [p for server, pids in server_pids.items() for p in pids] # keep track of the pids yeiled back as killed for all servers killed_pids = set() kill_wait = kwargs.get('kill_wait', KILL_WAIT) for server, killed_pid in watch_server_pids(server_pids, interval=kill_wait, **kwargs): print _("%s (%s) appears to have stopped") % (server, killed_pid) killed_pids.add(killed_pid) if not killed_pids.symmetric_difference(signaled_pids): # all proccesses have been stopped return 0 # reached interval n watch_pids w/o killing all servers for server, pids in server_pids.items(): if not killed_pids.issuperset(pids): # some pids of this server were not killed print _('Waited %s seconds for %s to die; giving up') % ( kill_wait, server) return 1 @command def kill(self, **kwargs): """stop a server (no error if not running) """ status = self.stop(**kwargs) kwargs['quiet'] = True if status and not self.status(**kwargs): # only exit error if the server is still running return status return 0 @command def shutdown(self, **kwargs): """allow current requests to finish on supporting servers """ kwargs['graceful'] = True status = 0 status += self.stop(**kwargs) return status @command def restart(self, **kwargs): """stops then restarts server """ status = 0 status += self.stop(**kwargs) status += self.start(**kwargs) return status @command def reload(self, **kwargs): """graceful shutdown then restart on supporting servers """ kwargs['graceful'] = True status = 0 for server in self.servers: m = Manager([server.server]) status += m.stop(**kwargs) status += m.start(**kwargs) return status @command def force_reload(self, **kwargs): """alias for reload """ return self.reload(**kwargs) def get_command(self, cmd): """Find and return the decorated method named like cmd :param cmd: the command to get, a string, if not found raises UnknownCommandError """ cmd = cmd.lower().replace('-', '_') try: f = getattr(self, cmd) except AttributeError: raise UnknownCommandError(cmd) if not hasattr(f, 'publicly_accessible'): raise UnknownCommandError(cmd) return f @classmethod def list_commands(cls): """Get all publicly accessible commands :returns: a list of string tuples (cmd, help), the method names who are decorated as commands """ get_method = lambda cmd: getattr(cls, cmd) return sorted([(x.replace('_', '-'), get_method(x).__doc__.strip()) for x in dir(cls) if getattr(get_method(x), 'publicly_accessible', False)]) def run_command(self, cmd, **kwargs): """Find the named command and run it :param cmd: the command name to run """ f = self.get_command(cmd) return f(**kwargs) class Server(object): """Manage operations on a server or group of servers of similar type :param server: name of server """ def __init__(self, server, run_dir=RUN_DIR): if '-' not in server: server = '%s-server' % server self.server = server.lower() self.type = server.rsplit('-', 1)[0] self.cmd = 'swift-%s' % server self.procs = [] self.run_dir = run_dir def __str__(self): return self.server def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(str(self))) def __hash__(self): return hash(str(self)) def __eq__(self, other): try: return self.server == other.server except AttributeError: return False def get_pid_file_name(self, conf_file): """Translate conf_file to a corresponding pid_file :param conf_file: an conf_file for this server, a string :returns: the pid_file for this conf_file """ return conf_file.replace( os.path.normpath(SWIFT_DIR), self.run_dir, 1).replace( '%s-server' % self.type, self.server, 1).replace( '.conf', '.pid', 1) def get_conf_file_name(self, pid_file): """Translate pid_file to a corresponding conf_file :param pid_file: a pid_file for this server, a string :returns: the conf_file for this pid_file """ if self.server in STANDALONE_SERVERS: return pid_file.replace( os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace( '.pid', '.conf', 1) else: return pid_file.replace( os.path.normpath(self.run_dir), SWIFT_DIR, 1).replace( self.server, '%s-server' % self.type, 1).replace( '.pid', '.conf', 1) def conf_files(self, **kwargs): """Get conf files for this server :param: number, if supplied will only lookup the nth server :returns: list of conf files """ if self.server in STANDALONE_SERVERS: found_conf_files = search_tree(SWIFT_DIR, self.server + '*', '.conf', dir_ext='.conf.d') else: found_conf_files = search_tree(SWIFT_DIR, '%s-server*' % self.type, '.conf', dir_ext='.conf.d') number = kwargs.get('number') if number: try: conf_files = [found_conf_files[number - 1]] except IndexError: conf_files = [] else: conf_files = found_conf_files if not conf_files: # maybe there's a config file(s) out there, but I couldn't find it! if not kwargs.get('quiet'): print _('Unable to locate config %sfor %s') % ( ('number %s ' % number if number else ''), self.server) if kwargs.get('verbose') and not kwargs.get('quiet'): if found_conf_files: print _('Found configs:') for i, conf_file in enumerate(found_conf_files): print ' %d) %s' % (i + 1, conf_file) return conf_files def pid_files(self, **kwargs): """Get pid files for this server :param: number, if supplied will only lookup the nth server :returns: list of pid files """ pid_files = search_tree(self.run_dir, '%s*' % self.server) if kwargs.get('number', 0): conf_files = self.conf_files(**kwargs) # filter pid_files to match the index of numbered conf_file pid_files = [pid_file for pid_file in pid_files if self.get_conf_file_name(pid_file) in conf_files] return pid_files def iter_pid_files(self, **kwargs): """Generator, yields (pid_file, pids) """ for pid_file in self.pid_files(**kwargs): yield pid_file, int(open(pid_file).read().strip()) def signal_pids(self, sig, **kwargs): """Send a signal to pids for this server :param sig: signal to send :returns: a dict mapping pids (ints) to pid_files (paths) """ pids = {} for pid_file, pid in self.iter_pid_files(**kwargs): try: if sig != signal.SIG_DFL: print _('Signal %s pid: %s signal: %s') % (self.server, pid, sig) os.kill(pid, sig) except OSError as e: if e.errno == errno.ESRCH: # pid does not exist if kwargs.get('verbose'): print _("Removing stale pid file %s") % pid_file remove_file(pid_file) elif e.errno == errno.EPERM: print _("No permission to signal PID %d") % pid else: # process exists pids[pid] = pid_file return pids def get_running_pids(self, **kwargs): """Get running pids :returns: a dict mapping pids (ints) to pid_files (paths) """ return self.signal_pids(signal.SIG_DFL, **kwargs) # send noop def kill_running_pids(self, **kwargs): """Kill running pids :param graceful: if True, attempt SIGHUP on supporting servers :returns: a dict mapping pids (ints) to pid_files (paths) """ graceful = kwargs.get('graceful') if graceful and self.server in GRACEFUL_SHUTDOWN_SERVERS: sig = signal.SIGHUP else: sig = signal.SIGTERM return self.signal_pids(sig, **kwargs) def status(self, pids=None, **kwargs): """Display status of server :param: pids, if not supplied pids will be populated automatically :param: number, if supplied will only lookup the nth server :returns: 1 if server is not running, 0 otherwise """ if pids is None: pids = self.get_running_pids(**kwargs) if not pids: number = kwargs.get('number', 0) if number: kwargs['quiet'] = True conf_files = self.conf_files(**kwargs) if conf_files: print _("%s #%d not running (%s)") % (self.server, number, conf_files[0]) else: print _("No %s running") % self.server return 1 for pid, pid_file in pids.items(): conf_file = self.get_conf_file_name(pid_file) print _("%s running (%s - %s)") % (self.server, pid, conf_file) return 0 def spawn(self, conf_file, once=False, wait=True, daemon=True, **kwargs): """Launch a subprocess for this server. :param conf_file: path to conf_file to use as first arg :param once: boolean, add once argument to command :param wait: boolean, if true capture stdout with a pipe :param daemon: boolean, if false ask server to log to console :returns : the pid of the spawned process """ args = [self.cmd, conf_file] if once: args.append('once') if not daemon: # ask the server to log to console args.append('verbose') # figure out what we're going to do with stdio if not daemon: # do nothing, this process is open until the spawns close anyway re_out = None re_err = None else: re_err = subprocess.STDOUT if wait: # we're going to need to block on this... re_out = subprocess.PIPE else: re_out = open(os.devnull, 'w+b') proc = subprocess.Popen(args, stdout=re_out, stderr=re_err) pid_file = self.get_pid_file_name(conf_file) write_file(pid_file, proc.pid) self.procs.append(proc) return proc.pid def wait(self, **kwargs): """ wait on spawned procs to start """ status = 0 for proc in self.procs: # wait for process to close its stdout output = proc.stdout.read() if kwargs.get('once', False): # if you don't want once to wait you can send it to the # background on the command line, I generally just run with # no-daemon anyway, but this is quieter proc.wait() if output: print output start = time.time() # wait for process to die (output may just be a warning) while time.time() - start < WARNING_WAIT: time.sleep(0.1) if proc.poll() is not None: status += proc.returncode break return status def interact(self, **kwargs): """ wait on spawned procs to terminate """ status = 0 for proc in self.procs: # wait for process to terminate proc.communicate() if proc.returncode: status += 1 return status def launch(self, **kwargs): """ Collect conf files and attempt to spawn the processes for this server """ conf_files = self.conf_files(**kwargs) if not conf_files: return [] pids = self.get_running_pids(**kwargs) already_started = False for pid, pid_file in pids.items(): conf_file = self.get_conf_file_name(pid_file) # for legacy compat you can't start other servers if one server is # already running (unless -n specifies which one you want), this # restriction could potentially be lifted, and launch could start # any unstarted instances if conf_file in conf_files: already_started = True print _("%s running (%s - %s)") % (self.server, pid, conf_file) elif not kwargs.get('number', 0): already_started = True print _("%s running (%s - %s)") % (self.server, pid, pid_file) if already_started: print _("%s already started...") % self.server return [] if self.server not in START_ONCE_SERVERS: kwargs['once'] = False pids = {} for conf_file in conf_files: if kwargs.get('once'): msg = _('Running %s once') % self.server else: msg = _('Starting %s') % self.server print '%s...(%s)' % (msg, conf_file) try: pid = self.spawn(conf_file, **kwargs) except OSError as e: if e.errno == errno.ENOENT: #TODO(clayg): should I check if self.cmd exists earlier? print _("%s does not exist") % self.cmd break else: raise pids[pid] = conf_file return pids def stop(self, **kwargs): """Send stop signals to pids for this server :returns: a dict mapping pids (ints) to pid_files (paths) """ return self.kill_running_pids(**kwargs) swift-1.13.1/swift/common/wsgi.py0000664000175400017540000006100312323703611020034 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """WSGI tools for use with swift.""" import errno import inspect import os import signal import time import mimetools from swift import gettext_ as _ from StringIO import StringIO import eventlet import eventlet.debug from eventlet import greenio, GreenPool, sleep, wsgi, listen from paste.deploy import loadwsgi from eventlet.green import socket, ssl from urllib import unquote from swift.common import utils, constraints from swift.common.swob import Request from swift.common.utils import capture_stdio, disable_fallocate, \ drop_privileges, get_logger, NullLogger, config_true_value, \ validate_configuration, get_hub, config_auto_int_value, \ CloseableChain # Set maximum line size of message headers to be accepted. wsgi.MAX_HEADER_LINE = constraints.MAX_HEADER_SIZE try: import multiprocessing CPU_COUNT = multiprocessing.cpu_count() or 1 except (ImportError, NotImplementedError): CPU_COUNT = 1 class NamedConfigLoader(loadwsgi.ConfigLoader): """ Patch paste.deploy's ConfigLoader so each context object will know what config section it came from. """ def get_context(self, object_type, name=None, global_conf=None): context = super(NamedConfigLoader, self).get_context( object_type, name=name, global_conf=global_conf) context.name = name return context loadwsgi.ConfigLoader = NamedConfigLoader class ConfigDirLoader(NamedConfigLoader): """ Read configuration from multiple files under the given path. """ def __init__(self, conf_dir): # parent class uses filename attribute when building error messages self.filename = conf_dir = conf_dir.strip() defaults = { 'here': os.path.normpath(os.path.abspath(conf_dir)), '__file__': os.path.abspath(conf_dir) } self.parser = loadwsgi.NicerConfigParser(conf_dir, defaults=defaults) self.parser.optionxform = str # Don't lower-case keys utils.read_conf_dir(self.parser, conf_dir) def _loadconfigdir(object_type, uri, path, name, relative_to, global_conf): if relative_to: path = os.path.normpath(os.path.join(relative_to, path)) loader = ConfigDirLoader(path) if global_conf: loader.update_defaults(global_conf, overwrite=False) return loader.get_context(object_type, name, global_conf) # add config_dir parsing to paste.deploy loadwsgi._loaders['config_dir'] = _loadconfigdir def wrap_conf_type(f): """ Wrap a function whos first argument is a paste.deploy style config uri, such that you can pass it an un-adorned raw filesystem path and the config directive (either config: or config_dir:) will be added automatically based on the type of filesystem entity at the given path (either a file or directory) before passing it through to the paste.deploy function. """ def wrapper(conf_path, *args, **kwargs): if os.path.isdir(conf_path): conf_type = 'config_dir' else: conf_type = 'config' conf_uri = '%s:%s' % (conf_type, conf_path) return f(conf_uri, *args, **kwargs) return wrapper appconfig = wrap_conf_type(loadwsgi.appconfig) def monkey_patch_mimetools(): """ mimetools.Message defaults content-type to "text/plain" This changes it to default to None, so we can detect missing headers. """ orig_parsetype = mimetools.Message.parsetype def parsetype(self): if not self.typeheader: self.type = None self.maintype = None self.subtype = None self.plisttext = '' else: orig_parsetype(self) parsetype.patched = True if not getattr(mimetools.Message.parsetype, 'patched', None): mimetools.Message.parsetype = parsetype def get_socket(conf, default_port=8080): """Bind socket to bind ip:port in conf :param conf: Configuration dict to read settings from :param default_port: port to use if not specified in conf :returns : a socket object as returned from socket.listen or ssl.wrap_socket if conf specifies cert_file """ bind_addr = (conf.get('bind_ip', '0.0.0.0'), int(conf.get('bind_port', default_port))) address_family = [addr[0] for addr in socket.getaddrinfo( bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM) if addr[0] in (socket.AF_INET, socket.AF_INET6)][0] sock = None bind_timeout = int(conf.get('bind_timeout', 30)) retry_until = time.time() + bind_timeout warn_ssl = False while not sock and time.time() < retry_until: try: sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)), family=address_family) if 'cert_file' in conf: warn_ssl = True sock = ssl.wrap_socket(sock, certfile=conf['cert_file'], keyfile=conf['key_file']) except socket.error as err: if err.args[0] != errno.EADDRINUSE: raise sleep(0.1) if not sock: raise Exception(_('Could not bind to %s:%s ' 'after trying for %s seconds') % ( bind_addr[0], bind_addr[1], bind_timeout)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # in my experience, sockets can hang around forever without keepalive sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) if hasattr(socket, 'TCP_KEEPIDLE'): sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) if warn_ssl: ssl_warning_message = _('WARNING: SSL should only be enabled for ' 'testing purposes. Use external SSL ' 'termination for a production deployment.') get_logger(conf).warning(ssl_warning_message) print(ssl_warning_message) return sock class RestrictedGreenPool(GreenPool): """ Works the same as GreenPool, but if the size is specified as one, then the spawn_n() method will invoke waitall() before returning to prevent the caller from doing any other work (like calling accept()). """ def __init__(self, size=1024): super(RestrictedGreenPool, self).__init__(size=size) self._rgp_do_wait = (size == 1) def spawn_n(self, *args, **kwargs): super(RestrictedGreenPool, self).spawn_n(*args, **kwargs) if self._rgp_do_wait: self.waitall() class PipelineWrapper(object): """ This class provides a number of utility methods for modifying the composition of a wsgi pipeline. """ def __init__(self, context): self.context = context def __contains__(self, entry_point_name): try: self.index(entry_point_name) return True except ValueError: return False def startswith(self, entry_point_name): """ Tests if the pipeline starts with the given entry point name. :param entry_point_name: entry point of middleware or app (Swift only) :returns: True if entry_point_name is first in pipeline, False otherwise """ try: first_ctx = self.context.filter_contexts[0] except IndexError: first_ctx = self.context.app_context return first_ctx.entry_point_name == entry_point_name def _format_for_display(self, ctx): if ctx.entry_point_name: return ctx.entry_point_name elif inspect.isfunction(ctx.object): # ctx.object is a reference to the actual filter_factory # function, so we pretty-print that. It's not the nice short # entry point, but it beats "". # # These happen when, instead of something like # # use = egg:swift#healthcheck # # you have something like this: # # paste.filter_factory = \ # swift.common.middleware.healthcheck:filter_factory return "%s:%s" % (inspect.getmodule(ctx.object).__name__, ctx.object.__name__) else: # No idea what this is return "" def __str__(self): parts = [self._format_for_display(ctx) for ctx in self.context.filter_contexts] parts.append(self._format_for_display(self.context.app_context)) return " ".join(parts) def create_filter(self, entry_point_name): """ Creates a context for a filter that can subsequently be added to a pipeline context. :param entry_point_name: entry point of the middleware (Swift only) :returns: a filter context """ spec = 'egg:swift#' + entry_point_name ctx = loadwsgi.loadcontext(loadwsgi.FILTER, spec, global_conf=self.context.global_conf) ctx.protocol = 'paste.filter_factory' return ctx def index(self, entry_point_name): """ Returns the first index of the given entry point name in the pipeline. Raises ValueError if the given module is not in the pipeline. """ for i, ctx in enumerate(self.context.filter_contexts): if ctx.entry_point_name == entry_point_name: return i raise ValueError("%s is not in pipeline" % (entry_point_name,)) def insert_filter(self, ctx, index=0): """ Inserts a filter module into the pipeline context. :param ctx: the context to be inserted :param index: (optional) index at which filter should be inserted in the list of pipeline filters. Default is 0, which means the start of the pipeline. """ self.context.filter_contexts.insert(index, ctx) def loadcontext(object_type, uri, name=None, relative_to=None, global_conf=None): add_conf_type = wrap_conf_type(lambda x: x) return loadwsgi.loadcontext(object_type, add_conf_type(uri), name=name, relative_to=relative_to, global_conf=global_conf) def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True): """ Loads a context from a config file, and if the context is a pipeline then presents the app with the opportunity to modify the pipeline. """ global_conf = global_conf or {} ctx = loadcontext(loadwsgi.APP, conf_file, global_conf=global_conf) if ctx.object_type.name == 'pipeline': # give app the opportunity to modify the pipeline context app = ctx.app_context.create() func = getattr(app, 'modify_wsgi_pipeline', None) if func and allow_modify_pipeline: func(PipelineWrapper(ctx)) return ctx.create() def run_server(conf, logger, sock, global_conf=None): # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to the timezone in which # the server first starts running in locations that periodically change # timezones. os.environ['TZ'] = time.strftime("%z", time.gmtime()) wsgi.HttpProtocol.default_request_version = "HTTP/1.0" # Turn off logging requests by the underlying WSGI software. wsgi.HttpProtocol.log_request = lambda *a: None # Redirect logging other messages by the underlying WSGI software. wsgi.HttpProtocol.log_message = \ lambda s, f, *a: logger.error('ERROR WSGI: ' + f % a) wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60) eventlet.hubs.use_hub(get_hub()) eventlet.patcher.monkey_patch(all=False, socket=True) eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no')) eventlet.debug.hub_exceptions(eventlet_debug) # utils.LogAdapter stashes name in server; fallback on unadapted loggers if not global_conf: if hasattr(logger, 'server'): log_name = logger.server else: log_name = logger.name global_conf = {'log_name': log_name} app = loadapp(conf['__file__'], global_conf=global_conf) max_clients = int(conf.get('max_clients', '1024')) pool = RestrictedGreenPool(size=max_clients) try: wsgi.server(sock, app, NullLogger(), custom_pool=pool) except socket.error as err: if err[0] != errno.EINVAL: raise pool.waitall() #TODO(clayg): pull more pieces of this to test more def run_wsgi(conf_path, app_section, *args, **kwargs): """ Runs the server using the specified number of workers. :param conf_path: Path to paste.deploy style configuration file/directory :param app_section: App name from conf file to load config from :returns: 0 if successful, nonzero otherwise """ # Load configuration, Set logger and Load request processor try: (conf, logger, log_name) = \ _initrp(conf_path, app_section, *args, **kwargs) except ConfigFileError as e: print e return 1 # bind to address and port sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) # remaining tasks should not require elevated privileges drop_privileges(conf.get('user', 'swift')) # Ensure the configuration and application can be loaded before proceeding. global_conf = {'log_name': log_name} if 'global_conf_callback' in kwargs: kwargs['global_conf_callback'](conf, global_conf) loadapp(conf_path, global_conf=global_conf) # set utils.FALLOCATE_RESERVE if desired reserve = int(conf.get('fallocate_reserve', 0)) if reserve > 0: utils.FALLOCATE_RESERVE = reserve # redirect errors to logger and close stdio capture_stdio(logger) worker_count = config_auto_int_value(conf.get('workers'), CPU_COUNT) # Useful for profiling [no forks]. if worker_count == 0: run_server(conf, logger, sock, global_conf=global_conf) return 0 def kill_children(*args): """Kills the entire process group.""" logger.error('SIGTERM received') signal.signal(signal.SIGTERM, signal.SIG_IGN) running[0] = False os.killpg(0, signal.SIGTERM) def hup(*args): """Shuts down the server, but allows running requests to complete""" logger.error('SIGHUP received') signal.signal(signal.SIGHUP, signal.SIG_IGN) running[0] = False running = [True] signal.signal(signal.SIGTERM, kill_children) signal.signal(signal.SIGHUP, hup) children = [] while running[0]: while len(children) < worker_count: pid = os.fork() if pid == 0: signal.signal(signal.SIGHUP, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) run_server(conf, logger, sock) logger.notice('Child %d exiting normally' % os.getpid()) return 0 else: logger.notice('Started child %s' % pid) children.append(pid) try: pid, status = os.wait() if os.WIFEXITED(status) or os.WIFSIGNALED(status): logger.error('Removing dead child %s' % pid) children.remove(pid) except OSError as err: if err.errno not in (errno.EINTR, errno.ECHILD): raise except KeyboardInterrupt: logger.notice('User quit') break greenio.shutdown_safe(sock) sock.close() logger.notice('Exited') return 0 class ConfigFileError(Exception): pass def _initrp(conf_path, app_section, *args, **kwargs): try: conf = appconfig(conf_path, name=app_section) except Exception as e: raise ConfigFileError("Error trying to load config from %s: %s" % (conf_path, e)) validate_configuration() # pre-configure logger log_name = conf.get('log_name', app_section) if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = get_logger(conf, log_name, log_to_console=kwargs.pop('verbose', False), log_route='wsgi') # disable fallocate if desired if config_true_value(conf.get('disable_fallocate', 'no')): disable_fallocate() monkey_patch_mimetools() return (conf, logger, log_name) def init_request_processor(conf_path, app_section, *args, **kwargs): """ Loads common settings from conf Sets the logger Loads the request processor :param conf_path: Path to paste.deploy style configuration file/directory :param app_section: App name from conf file to load config from :returns: the loaded application entry point :raises ConfigFileError: Exception is raised for config file error """ (conf, logger, log_name) = _initrp(conf_path, app_section, *args, **kwargs) app = loadapp(conf_path, global_conf={'log_name': log_name}) return (app, conf, logger, log_name) class WSGIContext(object): """ This class provides a means to provide context (scope) for a middleware filter to have access to the wsgi start_response results like the request status and headers. """ def __init__(self, wsgi_app): self.app = wsgi_app def _start_response(self, status, headers, exc_info=None): """ Saves response info without sending it to the remote client. Uses the same semantics as the usual WSGI start_response. """ self._response_status = status self._response_headers = headers self._response_exc_info = exc_info def _app_call(self, env): """ Ensures start_response has been called before returning. """ self._response_status = None self._response_headers = None self._response_exc_info = None resp = self.app(env, self._start_response) # if start_response has been called, just return the iter if self._response_status is not None: return resp resp = iter(resp) try: first_chunk = resp.next() except StopIteration: return iter([]) else: # We got a first_chunk return CloseableChain([first_chunk], resp) def _get_status_int(self): """ Returns the HTTP status int from the last called self._start_response result. """ return int(self._response_status.split(' ', 1)[0]) def _response_header_value(self, key): "Returns str of value for given header key or None" for h_key, val in self._response_headers: if h_key.lower() == key.lower(): return val return None def make_env(env, method=None, path=None, agent='Swift', query_string=None, swift_source=None): """ Returns a new fresh WSGI environment. :param env: The WSGI environment to base the new environment on. :param method: The new REQUEST_METHOD or None to use the original. :param path: The new path_info or none to use the original. path should NOT be quoted. When building a url, a Webob Request (in accordance with wsgi spec) will quote env['PATH_INFO']. url += quote(environ['PATH_INFO']) :param query_string: The new query_string or none to use the original. When building a url, a Webob Request will append the query string directly to the url. url += '?' + env['QUERY_STRING'] :param agent: The HTTP user agent to use; default 'Swift'. You can put %(orig)s in the agent to have it replaced with the original env's HTTP_USER_AGENT, such as '%(orig)s StaticWeb'. You also set agent to None to use the original env's HTTP_USER_AGENT or '' to have no HTTP_USER_AGENT. :param swift_source: Used to mark the request as originating out of middleware. Will be logged in proxy logs. :returns: Fresh WSGI environment. """ newenv = {} for name in ('eventlet.posthooks', 'HTTP_USER_AGENT', 'HTTP_HOST', 'PATH_INFO', 'QUERY_STRING', 'REMOTE_USER', 'REQUEST_METHOD', 'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT', 'HTTP_ORIGIN', 'HTTP_ACCESS_CONTROL_REQUEST_METHOD', 'SERVER_PROTOCOL', 'swift.cache', 'swift.source', 'swift.trans_id', 'swift.authorize_override', 'swift.authorize'): if name in env: newenv[name] = env[name] if method: newenv['REQUEST_METHOD'] = method if path: newenv['PATH_INFO'] = path newenv['SCRIPT_NAME'] = '' if query_string is not None: newenv['QUERY_STRING'] = query_string if agent: newenv['HTTP_USER_AGENT'] = ( agent % {'orig': env.get('HTTP_USER_AGENT', '')}).strip() elif agent == '' and 'HTTP_USER_AGENT' in newenv: del newenv['HTTP_USER_AGENT'] if swift_source: newenv['swift.source'] = swift_source newenv['wsgi.input'] = StringIO('') if 'SCRIPT_NAME' not in newenv: newenv['SCRIPT_NAME'] = '' return newenv def make_subrequest(env, method=None, path=None, body=None, headers=None, agent='Swift', swift_source=None, make_env=make_env): """ Makes a new swob.Request based on the current env but with the parameters specified. :param env: The WSGI environment to base the new request on. :param method: HTTP method of new request; default is from the original env. :param path: HTTP path of new request; default is from the original env. path should be compatible with what you would send to Request.blank. path should be quoted and it can include a query string. for example: '/a%20space?unicode_str%E8%AA%9E=y%20es' :param body: HTTP body of new request; empty by default. :param headers: Extra HTTP headers of new request; None by default. :param agent: The HTTP user agent to use; default 'Swift'. You can put %(orig)s in the agent to have it replaced with the original env's HTTP_USER_AGENT, such as '%(orig)s StaticWeb'. You also set agent to None to use the original env's HTTP_USER_AGENT or '' to have no HTTP_USER_AGENT. :param swift_source: Used to mark the request as originating out of middleware. Will be logged in proxy logs. :param make_env: make_subrequest calls this make_env to help build the swob.Request. :returns: Fresh swob.Request object. """ query_string = None path = path or '' if path and '?' in path: path, query_string = path.split('?', 1) newenv = make_env(env, method, path=unquote(path), agent=agent, query_string=query_string, swift_source=swift_source) if not headers: headers = {} if body: return Request.blank(path, environ=newenv, body=body, headers=headers) else: return Request.blank(path, environ=newenv, headers=headers) def make_pre_authed_env(env, method=None, path=None, agent='Swift', query_string=None, swift_source=None): """Same as :py:func:`make_env` but with preauthorization.""" newenv = make_env( env, method=method, path=path, agent=agent, query_string=query_string, swift_source=swift_source) newenv['swift.authorize'] = lambda req: None newenv['swift.authorize_override'] = True newenv['REMOTE_USER'] = '.wsgi.pre_authed' return newenv def make_pre_authed_request(env, method=None, path=None, body=None, headers=None, agent='Swift', swift_source=None): """Same as :py:func:`make_subrequest` but with preauthorization.""" return make_subrequest( env, method=method, path=path, body=body, headers=headers, agent=agent, swift_source=swift_source, make_env=make_pre_authed_env) swift-1.13.1/swift/common/daemon.py0000664000175400017540000000777712323703611020350 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import time import signal from re import sub import eventlet.debug from swift.common import utils class Daemon(object): """Daemon base class""" def __init__(self, conf): self.conf = conf self.logger = utils.get_logger(conf, log_route='daemon') def run_once(self, *args, **kwargs): """Override this to run the script once""" raise NotImplementedError('run_once not implemented') def run_forever(self, *args, **kwargs): """Override this to run forever""" raise NotImplementedError('run_forever not implemented') def run(self, once=False, **kwargs): """Run the daemon""" utils.validate_configuration() utils.drop_privileges(self.conf.get('user', 'swift')) utils.capture_stdio(self.logger, **kwargs) def kill_children(*args): signal.signal(signal.SIGTERM, signal.SIG_IGN) os.killpg(0, signal.SIGTERM) sys.exit() signal.signal(signal.SIGTERM, kill_children) if once: self.run_once(**kwargs) else: self.run_forever(**kwargs) def run_daemon(klass, conf_file, section_name='', once=False, **kwargs): """ Loads settings from conf, then instantiates daemon "klass" and runs the daemon with the specified once kwarg. The section_name will be derived from the daemon "klass" if not provided (e.g. ObjectReplicator => object-replicator). :param klass: Class to instantiate, subclass of common.daemon.Daemon :param conf_file: Path to configuration file :param section_name: Section name from conf file to load config from :param once: Passed to daemon run method """ # very often the config section_name is based on the class name # the None singleton will be passed through to readconf as is if section_name is '': section_name = sub(r'([a-z])([A-Z])', r'\1-\2', klass.__name__).lower() conf = utils.readconf(conf_file, section_name, log_name=kwargs.get('log_name')) # once on command line (i.e. daemonize=false) will over-ride config once = once or not utils.config_true_value(conf.get('daemonize', 'true')) # pre-configure logger if 'logger' in kwargs: logger = kwargs.pop('logger') else: logger = utils.get_logger(conf, conf.get('log_name', section_name), log_to_console=kwargs.pop('verbose', False), log_route=section_name) # disable fallocate if desired if utils.config_true_value(conf.get('disable_fallocate', 'no')): utils.disable_fallocate() # set utils.FALLOCATE_RESERVE if desired reserve = int(conf.get('fallocate_reserve', 0)) if reserve > 0: utils.FALLOCATE_RESERVE = reserve # By default, disable eventlet printing stacktraces eventlet_debug = utils.config_true_value(conf.get('eventlet_debug', 'no')) eventlet.debug.hub_exceptions(eventlet_debug) # Ensure TZ environment variable exists to avoid stat('/etc/localtime') on # some platforms. This locks in reported times to the timezone in which # the server first starts running in locations that periodically change # timezones. os.environ['TZ'] = time.strftime("%z", time.gmtime()) try: klass(conf).run(once=once, **kwargs) except KeyboardInterrupt: logger.info('User quit') logger.info('Exited') swift-1.13.1/swift/common/middleware/0000775000175400017540000000000012323703665020637 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/common/middleware/container_quotas.py0000664000175400017540000001434712323703611024567 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ The ``container_quotas`` middleware implements simple quotas that can be imposed on swift containers by a user with the ability to set container metadata, most likely the account administrator. This can be useful for limiting the scope of containers that are delegated to non-admin users, exposed to ``formpost`` uploads, or just as a self-imposed sanity check. Any object PUT operations that exceed these quotas return a 413 response (request entity too large) with a descriptive body. Quotas are subject to several limitations: eventual consistency, the timeliness of the cached container_info (60 second ttl by default), and it's unable to reject chunked transfer uploads that exceed the quota (though once the quota is exceeded, new chunked transfers will be refused). Quotas are set by adding meta values to the container, and are validated when set: +---------------------------------------------+-------------------------------+ |Metadata | Use | +=============================================+===============================+ | X-Container-Meta-Quota-Bytes | Maximum size of the | | | container, in bytes. | +---------------------------------------------+-------------------------------+ | X-Container-Meta-Quota-Count | Maximum object count of the | | | container. | +---------------------------------------------+-------------------------------+ """ from swift.common.constraints import check_copy_from_header from swift.common.http import is_success from swift.common.swob import Response, HTTPBadRequest, wsgify from swift.common.utils import register_swift_info from swift.proxy.controllers.base import get_container_info, get_object_info class ContainerQuotaMiddleware(object): def __init__(self, app, *args, **kwargs): self.app = app def bad_response(self, req, container_info): # 401 if the user couldn't have PUT this object in the first place. # This prevents leaking the container's existence to unauthed users. if 'swift.authorize' in req.environ: req.acl = container_info['write_acl'] aresp = req.environ['swift.authorize'](req) if aresp: return aresp return Response(status=413, body='Upload exceeds quota.') @wsgify def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method in ('PUT', 'COPY'): container_info = None if req.method == 'PUT': container_info = get_container_info( req.environ, self.app, swift_source='CQ') if req.method == 'COPY' and 'Destination' in req.headers: dest = req.headers.get('Destination').lstrip('/') path_info = req.environ['PATH_INFO'] req.environ['PATH_INFO'] = "/%s/%s/%s" % ( version, account, dest) try: container_info = get_container_info( req.environ, self.app, swift_source='CQ') finally: req.environ['PATH_INFO'] = path_info if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) if 'x-copy-from' in req.headers or req.method == 'COPY': if 'x-copy-from' in req.headers: container, obj = check_copy_from_header(req) path = '/%s/%s/%s/%s' % (version, account, container, obj) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app def filter_factory(global_conf, **local_conf): register_swift_info('container_quotas') def container_quota_filter(app): return ContainerQuotaMiddleware(app) return container_quota_filter swift-1.13.1/swift/common/middleware/keystoneauth.py0000664000175400017540000003317712323703611023736 0ustar jenkinsjenkins00000000000000# Copyright 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from swift.common import utils as swift_utils from swift.common.middleware import acl as swift_acl from swift.common.swob import HTTPNotFound, HTTPForbidden, HTTPUnauthorized from swift.common.utils import register_swift_info class KeystoneAuth(object): """Swift middleware to Keystone authorization system. In Swift's proxy-server.conf add this middleware to your pipeline:: [pipeline:main] pipeline = catch_errors cache authtoken keystoneauth proxy-server Make sure you have the authtoken middleware before the keystoneauth middleware. The authtoken middleware will take care of validating the user and keystoneauth will authorize access. The authtoken middleware is shipped directly with keystone it does not have any other dependences than itself so you can either install it by copying the file directly in your python path or by installing keystone. If support is required for unvalidated users (as with anonymous access) or for formpost/staticweb/tempurl middleware, authtoken will need to be configured with ``delay_auth_decision`` set to true. See the Keystone documentation for more detail on how to configure the authtoken middleware. In proxy-server.conf you will need to have the setting account auto creation to true:: [app:proxy-server] account_autocreate = true And add a swift authorization filter section, such as:: [filter:keystoneauth] use = egg:swift#keystoneauth operator_roles = admin, swiftoperator This maps tenants to account in Swift. The user whose able to give ACL / create Containers permissions will be the one that are inside the ``operator_roles`` setting which by default includes the admin and the swiftoperator roles. If you need to have a different reseller_prefix to be able to mix different auth servers you can configure the option ``reseller_prefix`` in your keystoneauth entry like this:: reseller_prefix = NEWAUTH :param app: The next WSGI app in the pipeline :param conf: The dict of configuration values """ def __init__(self, app, conf): self.app = app self.conf = conf self.logger = swift_utils.get_logger(conf, log_route='keystoneauth') self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.operator_roles = conf.get('operator_roles', 'admin, swiftoperator').lower() self.reseller_admin_role = conf.get('reseller_admin_role', 'ResellerAdmin').lower() config_is_admin = conf.get('is_admin', "false").lower() self.is_admin = swift_utils.config_true_value(config_is_admin) config_overrides = conf.get('allow_overrides', 't').lower() self.allow_overrides = swift_utils.config_true_value(config_overrides) def __call__(self, environ, start_response): identity = self._keystone_identity(environ) # Check if one of the middleware like tempurl or formpost have # set the swift.authorize_override environ and want to control the # authentication if (self.allow_overrides and environ.get('swift.authorize_override', False)): msg = 'Authorizing from an overriding middleware (i.e: tempurl)' self.logger.debug(msg) return self.app(environ, start_response) if identity: self.logger.debug('Using identity: %r', identity) environ['keystone.identity'] = identity environ['REMOTE_USER'] = identity.get('tenant') environ['swift.authorize'] = self.authorize user_roles = (r.lower() for r in identity.get('roles', [])) if self.reseller_admin_role in user_roles: environ['reseller_request'] = True else: self.logger.debug('Authorizing as anonymous') environ['swift.authorize'] = self.authorize_anonymous environ['swift.clean_acl'] = swift_acl.clean_acl return self.app(environ, start_response) def _keystone_identity(self, environ): """Extract the identity from the Keystone auth component.""" # In next release, we would add user id in env['keystone.identity'] by # using _integral_keystone_identity to replace current # _keystone_identity. The purpose of keeping it in this release it for # back compatibility. if environ.get('HTTP_X_IDENTITY_STATUS') != 'Confirmed': return roles = [] if 'HTTP_X_ROLES' in environ: roles = environ['HTTP_X_ROLES'].split(',') identity = {'user': environ.get('HTTP_X_USER_NAME'), 'tenant': (environ.get('HTTP_X_TENANT_ID'), environ.get('HTTP_X_TENANT_NAME')), 'roles': roles} return identity def _integral_keystone_identity(self, environ): """Extract the identity from the Keystone auth component.""" if environ.get('HTTP_X_IDENTITY_STATUS') != 'Confirmed': return roles = [] if 'HTTP_X_ROLES' in environ: roles = environ['HTTP_X_ROLES'].split(',') identity = {'user': (environ.get('HTTP_X_USER_ID'), environ.get('HTTP_X_USER_NAME')), 'tenant': (environ.get('HTTP_X_TENANT_ID'), environ.get('HTTP_X_TENANT_NAME')), 'roles': roles} return identity def _get_account_for_tenant(self, tenant_id): return '%s%s' % (self.reseller_prefix, tenant_id) def _reseller_check(self, account, tenant_id): """Check reseller prefix.""" return account == self._get_account_for_tenant(tenant_id) def _authorize_cross_tenant(self, user_id, user_name, tenant_id, tenant_name, roles): """Check cross-tenant ACLs. Match tenant:user, tenant and user could be its id, name or '*' :param user_id: The user id from the identity token. :param user_name: The user name from the identity token. :param tenant_id: The tenant ID from the identity token. :param tenant_name: The tenant name from the identity token. :param roles: The given container ACL. :returns: matched string if tenant(name/id/*):user(name/id/*) matches the given ACL. None otherwise. """ for tenant in [tenant_id, tenant_name, '*']: for user in [user_id, user_name, '*']: s = '%s:%s' % (tenant, user) if s in roles: return s return None def authorize(self, req): env = req.environ env_identity = self._integral_keystone_identity(env) tenant_id, tenant_name = env_identity['tenant'] user_id, user_name = env_identity['user'] referrers, roles = swift_acl.parse_acl(getattr(req, 'acl', None)) #allow OPTIONS requests to proceed as normal if req.method == 'OPTIONS': return try: part = req.split_path(1, 4, True) version, account, container, obj = part except ValueError: return HTTPNotFound(request=req) user_roles = [r.lower() for r in env_identity.get('roles', [])] # Give unconditional access to a user with the reseller_admin # role. if self.reseller_admin_role in user_roles: msg = 'User %s has reseller admin authorizing' self.logger.debug(msg, tenant_id) req.environ['swift_owner'] = True return # If we are not reseller admin and user is trying to delete its own # account then deny it. if not container and not obj and req.method == 'DELETE': # User is not allowed to issue a DELETE on its own account msg = 'User %s:%s is not allowed to delete its own account' self.logger.debug(msg, tenant_name, user_name) return self.denied_response(req) # cross-tenant authorization matched_acl = self._authorize_cross_tenant(user_id, user_name, tenant_id, tenant_name, roles) if matched_acl is not None: log_msg = 'user %s allowed in ACL authorizing.' self.logger.debug(log_msg, matched_acl) return acl_authorized = self._authorize_unconfirmed_identity(req, obj, referrers, roles) if acl_authorized: return # Check if a user tries to access an account that does not match their # token if not self._reseller_check(account, tenant_id): log_msg = 'tenant mismatch: %s != %s' self.logger.debug(log_msg, account, tenant_id) return self.denied_response(req) # Check the roles the user is belonging to. If the user is # part of the role defined in the config variable # operator_roles (like admin) then it will be # promoted as an admin of the account/tenant. for role in self.operator_roles.split(','): role = role.strip() if role in user_roles: log_msg = 'allow user with role %s as account admin' self.logger.debug(log_msg, role) req.environ['swift_owner'] = True return # If user is of the same name of the tenant then make owner of it. if self.is_admin and user_name == tenant_name: self.logger.warning("the is_admin feature has been deprecated " "and will be removed in the future " "update your config file") req.environ['swift_owner'] = True return if acl_authorized is not None: return self.denied_response(req) # Check if we have the role in the userroles and allow it for user_role in user_roles: if user_role in (r.lower() for r in roles): log_msg = 'user %s:%s allowed in ACL: %s authorizing' self.logger.debug(log_msg, tenant_name, user_name, user_role) return return self.denied_response(req) def authorize_anonymous(self, req): """ Authorize an anonymous request. :returns: None if authorization is granted, an error page otherwise. """ try: part = req.split_path(1, 4, True) version, account, container, obj = part except ValueError: return HTTPNotFound(request=req) #allow OPTIONS requests to proceed as normal if req.method == 'OPTIONS': return is_authoritative_authz = (account and account.startswith(self.reseller_prefix)) if not is_authoritative_authz: return self.denied_response(req) referrers, roles = swift_acl.parse_acl(getattr(req, 'acl', None)) authorized = self._authorize_unconfirmed_identity(req, obj, referrers, roles) if not authorized: return self.denied_response(req) def _authorize_unconfirmed_identity(self, req, obj, referrers, roles): """" Perform authorization for access that does not require a confirmed identity. :returns: A boolean if authorization is granted or denied. None if a determination could not be made. """ # Allow container sync. if (req.environ.get('swift_sync_key') and (req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None)) and 'x-timestamp' in req.headers): log_msg = 'allowing proxy %s for container-sync' self.logger.debug(log_msg, req.remote_addr) return True # Check if referrer is allowed. if swift_acl.referrer_allowed(req.referer, referrers): if obj or '.rlistings' in roles: log_msg = 'authorizing %s via referer ACL' self.logger.debug(log_msg, req.referrer) return True return False def denied_response(self, req): """Deny WSGI Response. Returns a standard WSGI response callable with the status of 403 or 401 depending on whether the REMOTE_USER is set or not. """ if req.remote_user: return HTTPForbidden(request=req) else: return HTTPUnauthorized(request=req) def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('keystoneauth') def auth_filter(app): return KeystoneAuth(app, conf) return auth_filter swift-1.13.1/swift/common/middleware/bulk.py0000664000175400017540000006443712323703614022156 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import tarfile from urllib import quote, unquote from xml.sax import saxutils from time import time from eventlet import sleep import zlib from swift.common.swob import Request, HTTPBadGateway, \ HTTPCreated, HTTPBadRequest, HTTPNotFound, HTTPUnauthorized, HTTPOk, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \ HTTPLengthRequired, HTTPException, HTTPServerError, wsgify from swift.common.utils import json, get_logger, register_swift_info from swift.common.constraints import check_utf8, MAX_FILE_SIZE from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT from swift.common.constraints import MAX_OBJECT_NAME_LENGTH, \ MAX_CONTAINER_NAME_LENGTH MAX_PATH_LENGTH = MAX_OBJECT_NAME_LENGTH + MAX_CONTAINER_NAME_LENGTH + 2 class CreateContainerError(Exception): def __init__(self, msg, status_int, status): self.status_int = status_int self.status = status Exception.__init__(self, msg) ACCEPTABLE_FORMATS = ['text/plain', 'application/json', 'application/xml', 'text/xml'] def get_response_body(data_format, data_dict, error_list): """ Returns a properly formatted response body according to format. Handles json and xml, otherwise will return text/plain. Note: xml response does not include xml declaration. :params data_format: resulting format :params data_dict: generated data about results. :params error_list: list of quoted filenames that failed """ if data_format == 'application/json': data_dict['Errors'] = error_list return json.dumps(data_dict) if data_format and data_format.endswith('/xml'): output = '\n' for key in sorted(data_dict): xml_key = key.replace(' ', '_').lower() output += '<%s>%s\n' % (xml_key, data_dict[key], xml_key) output += '\n' output += '\n'.join( ['' '%s%s' '' % (saxutils.escape(name), status) for name, status in error_list]) output += '\n\n' return output output = '' for key in sorted(data_dict): output += '%s: %s\n' % (key, data_dict[key]) output += 'Errors:\n' output += '\n'.join( ['%s, %s' % (name, status) for name, status in error_list]) return output class Bulk(object): """ Middleware that will do many operations on a single request. Extract Archive: Expand tar files into a swift account. Request must be a PUT with the query parameter ?extract-archive=format specifying the format of archive file. Accepted formats are tar, tar.gz, and tar.bz2. For a PUT to the following url: /v1/AUTH_Account/$UPLOAD_PATH?extract-archive=tar.gz UPLOAD_PATH is where the files will be expanded to. UPLOAD_PATH can be a container, a pseudo-directory within a container, or an empty string. The destination of a file in the archive will be built as follows: /v1/AUTH_Account/$UPLOAD_PATH/$FILE_PATH Where FILE_PATH is the file name from the listing in the tar file. If the UPLOAD_PATH is an empty string, containers will be auto created accordingly and files in the tar that would not map to any container (files in the base directory) will be ignored. Only regular files will be uploaded. Empty directories, symlinks, etc will not be uploaded. The response from bulk operations functions differently from other swift responses. This is because a short request body sent from the client could result in many operations on the proxy server and precautions need to be made to prevent the request from timing out due to lack of activity. To this end, the client will always receive a 200 OK response, regardless of the actual success of the call. The body of the response must be parsed to determine the actual success of the operation. In addition to this the client may receive zero or more whitespace characters prepended to the actual response body while the proxy server is completing the request. The format of the response body defaults to text/plain but can be either json or xml depending on the Accept header. Acceptable formats are text/plain, application/json, application/xml, and text/xml. An example body is as follows: {"Response Status": "201 Created", "Response Body": "", "Errors": [], "Number Files Created": 10} If all valid files were uploaded successfully the Response Status will be 201 Created. If any files failed to be created the response code corresponds to the subrequest's error. Possible codes are 400, 401, 502 (on server errors), etc. In both cases the response body will specify the number of files successfully uploaded and a list of the files that failed. There are proxy logs created for each file (which becomes a subrequest) in the tar. The subrequest's proxy log will have a swift.source set to "EA" the log's content length will reflect the unzipped size of the file. If double proxy-logging is used the leftmost logger will not have a swift.source set and the content length will reflect the size of the payload sent to the proxy (the unexpanded size of the tar.gz). Bulk Delete: Will delete multiple objects or containers from their account with a single request. Responds to POST requests with query parameter ?bulk-delete set. The request url is your storage url. The Content-Type should be set to text/plain. The body of the POST request will be a newline separated list of url encoded objects to delete. You can delete 10,000 (configurable) objects per request. The objects specified in the POST request body must be URL encoded and in the form: /container_name/obj_name or for a container (which must be empty at time of delete) /container_name The response is similar to extract archive as in every response will be a 200 OK and you must parse the response body for actual results. An example response is: {"Number Not Found": 0, "Response Status": "200 OK", "Response Body": "", "Errors": [], "Number Deleted": 6} If all items were successfully deleted (or did not exist), the Response Status will be 200 OK. If any failed to delete, the response code corresponds to the subrequest's error. Possible codes are 400, 401, 502 (on server errors), etc. In all cases the response body will specify the number of items successfully deleted, not found, and a list of those that failed. The return body will be formatted in the way specified in the request's Accept header. Acceptable formats are text/plain, application/json, application/xml, and text/xml. There are proxy logs created for each object or container (which becomes a subrequest) that is deleted. The subrequest's proxy log will have a swift.source set to "BD" the log's content length of 0. If double proxy-logging is used the leftmost logger will not have a swift.source set and the content length will reflect the size of the payload sent to the proxy (the list of objects/containers to be deleted). """ def __init__(self, app, conf, max_containers_per_extraction=10000, max_failed_extractions=1000, max_deletes_per_request=10000, max_failed_deletes=1000, yield_frequency=10, retry_count=0, retry_interval=1.5, logger=None): self.app = app self.logger = logger or get_logger(conf, log_route='bulk') self.max_containers = max_containers_per_extraction self.max_failed_extractions = max_failed_extractions self.max_failed_deletes = max_failed_deletes self.max_deletes_per_request = max_deletes_per_request self.yield_frequency = yield_frequency self.retry_count = retry_count self.retry_interval = retry_interval def create_container(self, req, container_path): """ Checks if the container exists and if not try to create it. :params container_path: an unquoted path to a container to be created :returns: True if created container, False if container exists :raises: CreateContainerError when unable to create container """ new_env = req.environ.copy() new_env['PATH_INFO'] = container_path new_env['swift.source'] = 'EA' new_env['REQUEST_METHOD'] = 'HEAD' head_cont_req = Request.blank(container_path, environ=new_env) resp = head_cont_req.get_response(self.app) if resp.is_success: return False if resp.status_int == 404: new_env = req.environ.copy() new_env['PATH_INFO'] = container_path new_env['swift.source'] = 'EA' new_env['REQUEST_METHOD'] = 'PUT' create_cont_req = Request.blank(container_path, environ=new_env) resp = create_cont_req.get_response(self.app) if resp.is_success: return True raise CreateContainerError( "Create Container Failed: " + container_path, resp.status_int, resp.status) def get_objs_to_delete(self, req): """ Will populate objs_to_delete with data from request input. :params req: a Swob request :returns: a list of the contents of req.body when separated by newline. :raises: HTTPException on failures """ line = '' data_remaining = True objs_to_delete = [] if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) while data_remaining: if '\n' in line: obj_to_delete, line = line.split('\n', 1) obj_to_delete = obj_to_delete.strip() objs_to_delete.append( {'name': unquote(obj_to_delete)}) else: data = req.body_file.read(MAX_PATH_LENGTH) if data: line += data else: data_remaining = False obj_to_delete = line.strip() if obj_to_delete: objs_to_delete.append( {'name': unquote(obj_to_delete)}) if len(objs_to_delete) > self.max_deletes_per_request: raise HTTPRequestEntityTooLarge( 'Maximum Bulk Deletes: %d per request' % self.max_deletes_per_request) if len(line) > MAX_PATH_LENGTH * 2: raise HTTPBadRequest('Invalid File Name') return objs_to_delete def handle_delete_iter(self, req, objs_to_delete=None, user_agent='BulkDelete', swift_source='BD', out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will delete the objects specified in request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params objs_to_delete: a list of dictionaries that specifies the objects to be deleted. If None, uses self.get_objs_to_delete to query request. """ last_yield = time() separator = '' failed_files = [] resp_dict = {'Response Status': HTTPOk().status, 'Response Body': '', 'Number Deleted': 0, 'Number Not Found': 0} try: if not out_content_type: raise HTTPNotAcceptable(request=req) if out_content_type.endswith('/xml'): yield '\n' try: vrs, account, _junk = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) incoming_format = req.headers.get('Content-Type') if incoming_format and \ not incoming_format.startswith('text/plain'): # For now only accept newline separated object names raise HTTPNotAcceptable(request=req) if objs_to_delete is None: objs_to_delete = self.get_objs_to_delete(req) failed_file_response = {'type': HTTPBadRequest} req.environ['eventlet.minimum_write_chunk_size'] = 0 for obj_to_delete in objs_to_delete: if last_yield + self.yield_frequency < time(): separator = '\r\n\r\n' last_yield = time() yield ' ' obj_name = obj_to_delete['name'] if not obj_name: continue if len(failed_files) >= self.max_failed_deletes: raise HTTPBadRequest('Max delete failures exceeded') if obj_to_delete.get('error'): if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 else: failed_files.append([quote(obj_name), obj_to_delete['error']['message']]) continue delete_path = '/'.join(['', vrs, account, obj_name.lstrip('/')]) if not check_utf8(delete_path): failed_files.append([quote(obj_name), HTTPPreconditionFailed().status]) continue new_env = req.environ.copy() new_env['PATH_INFO'] = delete_path del(new_env['wsgi.input']) new_env['CONTENT_LENGTH'] = 0 new_env['REQUEST_METHOD'] = 'DELETE' new_env['HTTP_USER_AGENT'] = \ '%s %s' % (req.environ.get('HTTP_USER_AGENT'), user_agent) new_env['swift.source'] = swift_source self._process_delete(delete_path, obj_name, new_env, resp_dict, failed_files, failed_file_response) if failed_files: resp_dict['Response Status'] = \ failed_file_response['type']().status elif not (resp_dict['Number Deleted'] or resp_dict['Number Not Found']): resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid bulk delete.' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body except Exception: self.logger.exception('Error in bulk delete.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body(out_content_type, resp_dict, failed_files) def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = {'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0} failed_files = [] last_yield = time() separator = '' containers_accessed = set() try: if not out_content_type: raise HTTPNotAcceptable(request=req) if out_content_type.endswith('/xml'): yield '\n' if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest req.environ['eventlet.minimum_write_chunk_size'] = 0 containers_created = 0 while True: if last_yield + self.yield_frequency < time(): separator = '\r\n\r\n' last_yield = time() yield ' ' tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join( ['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not check_utf8(destination): failed_files.append( [quote(obj_path[:MAX_PATH_LENGTH]), HTTPPreconditionFailed().status]) continue if tar_info.size > MAX_FILE_SIZE: failed_files.append([ quote(obj_path[:MAX_PATH_LENGTH]), HTTPRequestEntityTooLarge().status]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ quote(cont_path[:MAX_PATH_LENGTH]), err.status] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ quote(obj_path[:MAX_PATH_LENGTH]), HTTPBadRequest().status]) continue tar_file = tar.extractfile(tar_info) new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'PUT' new_env['wsgi.input'] = tar_file new_env['PATH_INFO'] = destination new_env['CONTENT_LENGTH'] = tar_info.size new_env['swift.source'] = 'EA' new_env['HTTP_USER_AGENT'] = \ '%s BulkExpand' % req.environ.get('HTTP_USER_AGENT') create_obj_req = Request.blank(destination, new_env) resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ quote(obj_path[:MAX_PATH_LENGTH]), HTTPUnauthorized().status]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ quote(obj_path[:MAX_PATH_LENGTH]), resp.status]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body( out_content_type, resp_dict, failed_files) def _process_delete(self, delete_path, obj_name, env, resp_dict, failed_files, failed_file_response, retry=0): delete_obj_req = Request.blank(delete_path, env) resp = delete_obj_req.get_response(self.app) if resp.status_int // 100 == 2: resp_dict['Number Deleted'] += 1 elif resp.status_int == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 elif resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([quote(obj_name), HTTPUnauthorized().status]) elif resp.status_int == HTTP_CONFLICT and \ self.retry_count > 0 and self.retry_count > retry: retry += 1 sleep(self.retry_interval ** retry) self._process_delete(delete_path, obj_name, env, resp_dict, failed_files, failed_file_response, retry) else: if resp.status_int // 100 == 5: failed_file_response['type'] = HTTPBadGateway failed_files.append([quote(obj_name), resp.status]) @wsgify def __call__(self, req): extract_type = req.params.get('extract-archive') resp = None if extract_type is not None and req.method == 'PUT': archive_type = { 'tar': '', 'tar.gz': 'gz', 'tar.bz2': 'bz2'}.get(extract_type.lower().strip('.')) if archive_type is not None: resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: resp.content_type = out_content_type resp.app_iter = self.handle_extract_iter( req, archive_type, out_content_type=out_content_type) else: resp = HTTPBadRequest("Unsupported archive format") if 'bulk-delete' in req.params and req.method in ['POST', 'DELETE']: resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: resp.content_type = out_content_type resp.app_iter = self.handle_delete_iter( req, out_content_type=out_content_type) return resp or self.app def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_containers_per_extraction = \ int(conf.get('max_containers_per_extraction', 10000)) max_failed_extractions = int(conf.get('max_failed_extractions', 1000)) max_deletes_per_request = int(conf.get('max_deletes_per_request', 10000)) max_failed_deletes = int(conf.get('max_failed_deletes', 1000)) yield_frequency = int(conf.get('yield_frequency', 10)) retry_count = int(conf.get('delete_container_retry_count', 0)) retry_interval = 1.5 register_swift_info( 'bulk_upload', max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions) register_swift_info( 'bulk_delete', max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes) def bulk_filter(app): return Bulk( app, conf, max_containers_per_extraction=max_containers_per_extraction, max_failed_extractions=max_failed_extractions, max_deletes_per_request=max_deletes_per_request, max_failed_deletes=max_failed_deletes, yield_frequency=yield_frequency, retry_count=retry_count, retry_interval=retry_interval) return bulk_filter swift-1.13.1/swift/common/middleware/crossdomain.py0000664000175400017540000000661612323703611023532 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.common.swob import Request, Response from swift.common.utils import register_swift_info class CrossDomainMiddleware(object): """ Cross domain middleware used to respond to requests for cross domain policy information. If the path is /crossdomain.xml it will respond with an xml cross domain policy document. This allows web pages hosted elsewhere to use client side technologies such as Flash, Java and Silverlight to interact with the Swift API. To enable this middleware, add it to the pipeline in your proxy-server.conf file. It should be added before any authentication (e.g., tempauth or keystone) middleware. In this example ellipsis (...) indicate other middleware you may have chosen to use:: [pipeline:main] pipeline = ... crossdomain ... authtoken ... proxy-server And add a filter section, such as:: [filter:crossdomain] use = egg:swift#crossdomain cross_domain_policy = For continuation lines, put some whitespace before the continuation text. Ensure you put a completely blank line to terminate the cross_domain_policy value. The cross_domain_policy name/value is optional. If omitted, the policy defaults as if you had specified:: cross_domain_policy = """ def __init__(self, app, conf, *args, **kwargs): self.app = app self.conf = conf default_domain_policy = '' self.cross_domain_policy = self.conf.get('cross_domain_policy', default_domain_policy) def GET(self, req): """Returns a 200 response with cross domain policy information """ body = '\n' \ '\n' \ '\n' \ '%s\n' \ '' % self.cross_domain_policy return Response(request=req, body=body, content_type="application/xml") def __call__(self, env, start_response): req = Request(env) if req.path == '/crossdomain.xml' and req.method == 'GET': return self.GET(req)(env, start_response) else: return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('crossdomain') def crossdomain_filter(app): return CrossDomainMiddleware(app, conf) return crossdomain_filter swift-1.13.1/swift/common/middleware/acl.py0000664000175400017540000002620112323703611021740 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.common.utils import urlparse, json def clean_acl(name, value): """ Returns a cleaned ACL header value, validating that it meets the formatting requirements for standard Swift ACL strings. The ACL format is:: [item[,item...]] Each item can be a group name to give access to or a referrer designation to grant or deny based on the HTTP Referer header. The referrer designation format is:: .r:[-]value The ``.r`` can also be ``.ref``, ``.referer``, or ``.referrer``; though it will be shortened to just ``.r`` for decreased character count usage. The value can be ``*`` to specify any referrer host is allowed access, a specific host name like ``www.example.com``, or if it has a leading period ``.`` or leading ``*.`` it is a domain name specification, like ``.example.com`` or ``*.example.com``. The leading minus sign ``-`` indicates referrer hosts that should be denied access. Referrer access is applied in the order they are specified. For example, .r:.example.com,.r:-thief.example.com would allow all hosts ending with .example.com except for the specific host thief.example.com. Example valid ACLs:: .r:* .r:*,.r:-.thief.com .r:*,.r:.example.com,.r:-thief.example.com .r:*,.r:-.thief.com,bobs_account,sues_account:sue bobs_account,sues_account:sue Example invalid ACLs:: .r: .r:- By default, allowing read access via .r will not allow listing objects in the container -- just retrieving objects from the container. To turn on listings, use the .rlistings directive. Also, .r designations aren't allowed in headers whose names include the word 'write'. ACLs that are "messy" will be cleaned up. Examples: ====================== ====================== Original Cleaned ---------------------- ---------------------- ``bob, sue`` ``bob,sue`` ``bob , sue`` ``bob,sue`` ``bob,,,sue`` ``bob,sue`` ``.referrer : *`` ``.r:*`` ``.ref:*.example.com`` ``.r:.example.com`` ``.r:*, .rlistings`` ``.r:*,.rlistings`` ====================== ====================== :param name: The name of the header being cleaned, such as X-Container-Read or X-Container-Write. :param value: The value of the header being cleaned. :returns: The value, cleaned of extraneous formatting. :raises ValueError: If the value does not meet the ACL formatting requirements; the error message will indicate why. """ name = name.lower() values = [] for raw_value in value.split(','): raw_value = raw_value.strip() if not raw_value: continue if ':' not in raw_value: values.append(raw_value) continue first, second = (v.strip() for v in raw_value.split(':', 1)) if not first or first[0] != '.': values.append(raw_value) elif first in ('.r', '.ref', '.referer', '.referrer'): if 'write' in name: raise ValueError('Referrers not allowed in write ACL: ' '%s' % repr(raw_value)) negate = False if second and second[0] == '-': negate = True second = second[1:].strip() if second and second != '*' and second[0] == '*': second = second[1:].strip() if not second or second == '.': raise ValueError('No host/domain value after referrer ' 'designation in ACL: %s' % repr(raw_value)) values.append('.r:%s%s' % ('-' if negate else '', second)) else: raise ValueError('Unknown designator %s in ACL: %s' % (repr(first), repr(raw_value))) return ','.join(values) def format_acl_v1(groups=None, referrers=None, header_name=None): """ Returns a standard Swift ACL string for the given inputs. Caller is responsible for ensuring that :referrers: parameter is only given if the ACL is being generated for X-Container-Read. (X-Container-Write and the account ACL headers don't support referrers.) :param groups: a list of groups (and/or members in most auth systems) to grant access :param referrers: a list of referrer designations (without the leading .r:) :param header_name: (optional) header name of the ACL we're preparing, for clean_acl; if None, returned ACL won't be cleaned :returns: a Swift ACL string for use in X-Container-{Read,Write}, X-Account-Access-Control, etc. """ groups, referrers = groups or [], referrers or [] referrers = ['.r:%s' % r for r in referrers] result = ','.join(groups + referrers) return (clean_acl(header_name, result) if header_name else result) def format_acl_v2(acl_dict): """ Returns a version-2 Swift ACL JSON string. HTTP headers for Version 2 ACLs have the following form: Header-Name: {"arbitrary":"json","encoded":"string"} JSON will be forced ASCII (containing six-char \uNNNN sequences rather than UTF-8; UTF-8 is valid JSON but clients vary in their support for UTF-8 headers), and without extraneous whitespace. Advantages over V1: forward compatibility (new keys don't cause parsing exceptions); Unicode support; no reserved words (you can have a user named .rlistings if you want). :param acl_dict: dict of arbitrary data to put in the ACL; see specific auth systems such as tempauth for supported values :returns: a JSON string which encodes the ACL """ return json.dumps(acl_dict, ensure_ascii=True, separators=(',', ':'), sort_keys=True) def format_acl(version=1, **kwargs): """ Compatibility wrapper to help migrate ACL syntax from version 1 to 2. Delegates to the appropriate version-specific format_acl method, defaulting to version 1 for backward compatibility. :param kwargs: keyword args appropriate for the selected ACL syntax version (see :func:`format_acl_v1` or :func:`format_acl_v2`) """ if version == 1: return format_acl_v1( groups=kwargs.get('groups'), referrers=kwargs.get('referrers'), header_name=kwargs.get('header_name')) elif version == 2: return format_acl_v2(kwargs.get('acl_dict')) raise ValueError("Invalid ACL version: %r" % version) def parse_acl_v1(acl_string): """ Parses a standard Swift ACL string into a referrers list and groups list. See :func:`clean_acl` for documentation of the standard Swift ACL format. :param acl_string: The standard Swift ACL string to parse. :returns: A tuple of (referrers, groups) where referrers is a list of referrer designations (without the leading .r:) and groups is a list of groups to allow access. """ referrers = [] groups = [] if acl_string: for value in acl_string.split(','): if value.startswith('.r:'): referrers.append(value[len('.r:'):]) else: groups.append(value) return referrers, groups def parse_acl_v2(data): """ Parses a version-2 Swift ACL string and returns a dict of ACL info. :param data: string containing the ACL data in JSON format :returns: A dict (possibly empty) containing ACL info, e.g.: {"groups": [...], "referrers": [...]} :returns: None if data is None, is not valid JSON or does not parse as a dict :returns: empty dictionary if data is an empty string """ if data is None: return None if data is '': return {} try: result = json.loads(data) return (result if type(result) is dict else None) except ValueError: return None def parse_acl(*args, **kwargs): """ Compatibility wrapper to help migrate ACL syntax from version 1 to 2. Delegates to the appropriate version-specific parse_acl method, attempting to determine the version from the types of args/kwargs. :param args: positional args for the selected ACL syntax version :param kwargs: keyword args for the selected ACL syntax version (see :func:`parse_acl_v1` or :func:`parse_acl_v2`) :returns: the return value of :func:`parse_acl_v1` or :func:`parse_acl_v2` """ version = kwargs.pop('version', None) if version in (1, None): return parse_acl_v1(*args) elif version == 2: return parse_acl_v2(*args, **kwargs) else: raise ValueError('Unknown ACL version: parse_acl(%r, %r)' % (args, kwargs)) def referrer_allowed(referrer, referrer_acl): """ Returns True if the referrer should be allowed based on the referrer_acl list (as returned by :func:`parse_acl`). See :func:`clean_acl` for documentation of the standard Swift ACL format. :param referrer: The value of the HTTP Referer header. :param referrer_acl: The list of referrer designations as returned by :func:`parse_acl`. :returns: True if the referrer should be allowed; False if not. """ allow = False if referrer_acl: rhost = urlparse(referrer or '').hostname or 'unknown' for mhost in referrer_acl: if mhost[0] == '-': mhost = mhost[1:] if mhost == rhost or (mhost[0] == '.' and rhost.endswith(mhost)): allow = False elif mhost == '*' or mhost == rhost or \ (mhost[0] == '.' and rhost.endswith(mhost)): allow = True return allow def acls_from_account_info(info): """ Extract the account ACLs from the given account_info, and return the ACLs. :param info: a dict of the form returned by get_account_info :returns: None (no ACL system metadata is set), or a dict of the form:: {'admin': [...], 'read-write': [...], 'read-only': [...]} :raises ValueError: on a syntactically invalid header """ acl = parse_acl( version=2, data=info.get('sysmeta', {}).get('core-access-control')) if acl is None: return None admin_members = acl.get('admin', []) readwrite_members = acl.get('read-write', []) readonly_members = acl.get('read-only', []) if not any((admin_members, readwrite_members, readonly_members)): return None return { 'admin': admin_members, 'read-write': readwrite_members, 'read-only': readonly_members, } swift-1.13.1/swift/common/middleware/catch_errors.py0000664000175400017540000000556012323703611023664 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift import gettext_ as _ from swift.common.swob import Request, HTTPServerError from swift.common.utils import get_logger, generate_trans_id from swift.common.wsgi import WSGIContext class CatchErrorsContext(WSGIContext): def __init__(self, app, logger, trans_id_suffix=''): super(CatchErrorsContext, self).__init__(app) self.logger = logger self.trans_id_suffix = trans_id_suffix def handle_request(self, env, start_response): trans_id = generate_trans_id(self.trans_id_suffix) env['swift.trans_id'] = trans_id self.logger.txn_id = trans_id try: # catch any errors in the pipeline resp = self._app_call(env) except: # noqa self.logger.exception(_('Error: An error occurred')) resp = HTTPServerError(request=Request(env), body='An error occurred', content_type='text/plain') resp.headers['X-Trans-Id'] = trans_id return resp(env, start_response) # make sure the response has the trans_id if self._response_headers is None: self._response_headers = [] self._response_headers.append(('X-Trans-Id', trans_id)) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp class CatchErrorMiddleware(object): """ Middleware that provides high-level error handling and ensures that a transaction id will be set for every request. """ def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='catch-errors') self.trans_id_suffix = conf.get('trans_id_suffix', '') def __call__(self, env, start_response): """ If used, this should be the first middleware in pipeline. """ context = CatchErrorsContext(self.app, self.logger, self.trans_id_suffix) return context.handle_request(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def except_filter(app): return CatchErrorMiddleware(app, conf) return except_filter swift-1.13.1/swift/common/middleware/dlo.py0000664000175400017540000003464112323703614021771 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os from ConfigParser import ConfigParser, NoSectionError, NoOptionError from hashlib import md5 from swift.common.constraints import CONTAINER_LISTING_LIMIT from swift.common.exceptions import ListingIterError from swift.common.http import is_success from swift.common.swob import Request, Response, \ HTTPRequestedRangeNotSatisfiable, HTTPBadRequest from swift.common.utils import get_logger, json, \ RateLimitedIterator, read_conf_dir, quote from swift.common.request_helpers import SegmentedIterable from swift.common.wsgi import WSGIContext, make_subrequest from urllib import unquote class GetContext(WSGIContext): def __init__(self, dlo, logger): super(GetContext, self).__init__(dlo.app) self.dlo = dlo self.logger = logger def _get_container_listing(self, req, version, account, container, prefix, marker=''): con_req = make_subrequest( req.environ, path='/'.join(['', version, account, container]), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO') con_req.query_string = 'format=json&prefix=%s' % quote(prefix) if marker: con_req.query_string += '&marker=%s' % quote(marker) con_resp = con_req.get_response(self.dlo.app) if not is_success(con_resp.status_int): return con_resp, None return None, json.loads(''.join(con_resp.app_iter)) def _segment_listing_iterator(self, req, version, account, container, prefix, segments, first_byte=None, last_byte=None): # It's sort of hokey that this thing takes in the first page of # segments as an argument, but we need to compute the etag and content # length from the first page, and it's better to have a hokey # interface than to make redundant requests. if first_byte is None: first_byte = 0 if last_byte is None: last_byte = float("inf") marker = '' while True: for segment in segments: seg_length = int(segment['bytes']) if first_byte >= seg_length: # don't need any bytes from this segment first_byte = max(first_byte - seg_length, -1) last_byte = max(last_byte - seg_length, -1) continue elif last_byte < 0: # no bytes are needed from this or any future segment break seg_name = segment['name'] if isinstance(seg_name, unicode): seg_name = seg_name.encode("utf-8") # (obj path, etag, size, first byte, last byte) yield ("/" + "/".join((version, account, container, seg_name)), # We deliberately omit the etag and size here; # SegmentedIterable will check size and etag if # specified, but we don't want it to. DLOs only care # that the objects' names match the specified prefix. None, None, (None if first_byte <= 0 else first_byte), (None if last_byte >= seg_length - 1 else last_byte)) first_byte = max(first_byte - seg_length, -1) last_byte = max(last_byte - seg_length, -1) if len(segments) < CONTAINER_LISTING_LIMIT: # a short page means that we're done with the listing break elif last_byte < 0: break marker = segments[-1]['name'] error_response, segments = self._get_container_listing( req, version, account, container, prefix, marker) if error_response: # we've already started sending the response body to the # client, so all we can do is raise an exception to make the # WSGI server close the connection early raise ListingIterError( "Got status %d listing container /%s/%s" % (error_response.status_int, account, container)) def get_or_head_response(self, req, x_object_manifest, response_headers=None): if response_headers is None: response_headers = self._response_headers container, obj_prefix = x_object_manifest.split('/', 1) container = unquote(container) obj_prefix = unquote(obj_prefix) # manifest might point to a different container req.acl = None version, account, _junk = req.split_path(2, 3, True) error_response, segments = self._get_container_listing( req, version, account, container, obj_prefix) if error_response: return error_response have_complete_listing = len(segments) < CONTAINER_LISTING_LIMIT first_byte = last_byte = None content_length = None if req.range and len(req.range.ranges) == 1: content_length = sum(o['bytes'] for o in segments) # This is a hack to handle suffix byte ranges (e.g. "bytes=-5"), # which we can't honor unless we have a complete listing. _junk, range_end = req.range.ranges_for_length(float("inf"))[0] # If this is all the segments, we know whether or not this # range request is satisfiable. # # Alternately, we may not have all the segments, but this range # falls entirely within the first page's segments, so we know # whether or not it's satisfiable. if have_complete_listing or range_end < content_length: byteranges = req.range.ranges_for_length(content_length) if not byteranges: return HTTPRequestedRangeNotSatisfiable(request=req) first_byte, last_byte = byteranges[0] # For some reason, swob.Range.ranges_for_length adds 1 to the # last byte's position. last_byte -= 1 else: # The range may or may not be satisfiable, but we can't tell # based on just one page of listing, and we're not going to go # get more pages because that would use up too many resources, # so we ignore the Range header and return the whole object. content_length = None req.range = None response_headers = [ (h, v) for h, v in response_headers if h.lower() not in ("content-length", "content-range")] if content_length is not None: # Here, we have to give swob a big-enough content length so that # it can compute the actual content length based on the Range # header. This value will not be visible to the client; swob will # substitute its own Content-Length. # # Note: if the manifest points to at least CONTAINER_LISTING_LIMIT # segments, this may be less than the sum of all the segments' # sizes. However, it'll still be greater than the last byte in the # Range header, so it's good enough for swob. response_headers.append(('Content-Length', str(content_length))) elif have_complete_listing: response_headers.append(('Content-Length', str(sum(o['bytes'] for o in segments)))) if have_complete_listing: response_headers = [(h, v) for h, v in response_headers if h.lower() != "etag"] etag = md5() for seg_dict in segments: etag.update(seg_dict['hash'].strip('"')) response_headers.append(('Etag', '"%s"' % etag.hexdigest())) listing_iter = RateLimitedIterator( self._segment_listing_iterator( req, version, account, container, obj_prefix, segments, first_byte=first_byte, last_byte=last_byte), self.dlo.rate_limit_segments_per_sec, limit_after=self.dlo.rate_limit_after_segment) resp = Response(request=req, headers=response_headers, conditional_response=True, app_iter=SegmentedIterable( req, self.dlo.app, listing_iter, ua_suffix="DLO MultipartGET", swift_source="DLO", name=req.path, logger=self.logger, max_get_time=self.dlo.max_get_time)) resp.app_iter.response = resp return resp def handle_request(self, req, start_response): """ Take a GET or HEAD request, and if it is for a dynamic large object manifest, return an appropriate response. Otherwise, simply pass it through. """ resp_iter = self._app_call(req.environ) # make sure this response is for a dynamic large object manifest for header, value in self._response_headers: if (header.lower() == 'x-object-manifest'): response = self.get_or_head_response(req, value) return response(req.environ, start_response) else: # Not a dynamic large object manifest; just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter class DynamicLargeObject(object): def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='dlo') # DLO functionality used to live in the proxy server, not middleware, # so let's try to go find config values in the proxy's config section # to ease cluster upgrades. self._populate_config_from_old_location(conf) self.max_get_time = int(conf.get('max_get_time', '86400')) self.rate_limit_after_segment = int(conf.get( 'rate_limit_after_segment', '10')) self.rate_limit_segments_per_sec = int(conf.get( 'rate_limit_segments_per_sec', '1')) def _populate_config_from_old_location(self, conf): if ('rate_limit_after_segment' in conf or 'rate_limit_segments_per_sec' in conf or 'max_get_time' in conf or '__file__' not in conf): return cp = ConfigParser() if os.path.isdir(conf['__file__']): read_conf_dir(cp, conf['__file__']) else: cp.read(conf['__file__']) try: pipe = cp.get("pipeline:main", "pipeline") except (NoSectionError, NoOptionError): return proxy_name = pipe.rsplit(None, 1)[-1] proxy_section = "app:" + proxy_name for setting in ('rate_limit_after_segment', 'rate_limit_segments_per_sec', 'max_get_time'): try: conf[setting] = cp.get(proxy_section, setting) except (NoSectionError, NoOptionError): pass def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) if ((req.method == 'GET' or req.method == 'HEAD') and req.params.get('multipart-manifest') != 'get'): return GetContext(self, self.logger).\ handle_request(req, start_response) elif req.method == 'PUT': error_response = self.validate_x_object_manifest_header( req, start_response) if error_response: return error_response(env, start_response) return self.app(env, start_response) def validate_x_object_manifest_header(self, req, start_response): """ Make sure that X-Object-Manifest is valid if present. """ if 'X-Object-Manifest' in req.headers: value = req.headers['X-Object-Manifest'] container = prefix = None try: container, prefix = value.split('/', 1) except ValueError: pass if not container or not prefix or '?' in value or '&' in value or \ prefix[0] == '/': return HTTPBadRequest( request=req, body=('X-Object-Manifest must be in the ' 'format container/prefix')) def copy_hook(self, inner_hook): def dlo_copy_hook(source_req, source_resp, sink_req): x_o_m = source_resp.headers.get('X-Object-Manifest') if x_o_m: if source_req.params.get('multipart-manifest') == 'get': # To copy the manifest, we let the copy proceed as normal, # but ensure that X-Object-Manifest is set on the new # object. sink_req.headers['X-Object-Manifest'] = x_o_m else: ctx = GetContext(self, self.logger) source_resp = ctx.get_or_head_response( source_req, x_o_m, source_resp.headers.items()) return inner_hook(source_req, source_resp, sink_req) return dlo_copy_hook def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def dlo_filter(app): return DynamicLargeObject(app, conf) return dlo_filter swift-1.13.1/swift/common/middleware/healthcheck.py0000664000175400017540000000426412323703611023451 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os from swift.common.swob import Request, Response class HealthCheckMiddleware(object): """ Healthcheck middleware used for monitoring. If the path is /healthcheck, it will respond 200 with "OK" as the body. If the optional config parameter "disable_path" is set, and a file is present at that path, it will respond 503 with "DISABLED BY FILE" as the body. """ def __init__(self, app, conf): self.app = app self.conf = conf self.disable_path = conf.get('disable_path', '') def GET(self, req): """Returns a 200 response with "OK" in the body.""" return Response(request=req, body="OK", content_type="text/plain") def DISABLED(self, req): """Returns a 503 response with "DISABLED BY FILE" in the body.""" return Response(request=req, status=503, body="DISABLED BY FILE", content_type="text/plain") def __call__(self, env, start_response): req = Request(env) try: if req.path == '/healthcheck': handler = self.GET if self.disable_path and os.path.exists(self.disable_path): handler = self.DISABLED return handler(req)(env, start_response) except UnicodeError: # definitely, this is not /healthcheck pass return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def healthcheck_filter(app): return HealthCheckMiddleware(app, conf) return healthcheck_filter swift-1.13.1/swift/common/middleware/name_check.py0000664000175400017540000001201412323703611023253 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. ''' Created on February 27, 2012 A filter that disallows any paths that contain defined forbidden characters or that exceed a defined length. Place early in the proxy-server pipeline after the left-most occurrence of the ``proxy-logging`` middleware (if present) and before the final ``proxy-logging`` middleware (if present) or the ``proxy-serer`` app itself, e.g.:: [pipeline:main] pipeline = catch_errors healthcheck proxy-logging name_check cache \ ratelimit tempauth sos proxy-logging proxy-server [filter:name_check] use = egg:swift#name_check forbidden_chars = '"`<> maximum_length = 255 There are default settings for forbidden_chars (FORBIDDEN_CHARS) and maximum_length (MAX_LENGTH) The filter returns HTTPBadRequest if path is invalid. @author: eamonn-otoole ''' import re from swift.common.utils import get_logger from urllib2 import unquote from swift.common.swob import Request, HTTPBadRequest FORBIDDEN_CHARS = "\'\"`<>" MAX_LENGTH = 255 FORBIDDEN_REGEXP = "/\./|/\.\./|/\.$|/\.\.$" class NameCheckMiddleware(object): def __init__(self, app, conf): self.app = app self.conf = conf self.forbidden_chars = self.conf.get('forbidden_chars', FORBIDDEN_CHARS) self.maximum_length = self.conf.get('maximum_length', MAX_LENGTH) self.forbidden_regexp = self.conf.get('forbidden_regexp', FORBIDDEN_REGEXP) if self.forbidden_regexp: self.forbidden_regexp_compiled = re.compile(self.forbidden_regexp) else: self.forbidden_regexp_compiled = None self.logger = get_logger(self.conf, log_route='name_check') def check_character(self, req): ''' Checks req.path for any forbidden characters Returns True if there are any forbidden characters Returns False if there aren't any forbidden characters ''' self.logger.debug("name_check: path %s" % req.path) self.logger.debug("name_check: self.forbidden_chars %s" % self.forbidden_chars) for c in unquote(req.path): if c in self.forbidden_chars: return True else: pass return False def check_length(self, req): ''' Checks that req.path doesn't exceed the defined maximum length Returns True if the length exceeds the maximum Returns False if the length is <= the maximum ''' length = len(unquote(req.path)) if length > self.maximum_length: return True else: return False def check_regexp(self, req): ''' Checks that req.path doesn't contain a substring matching regexps. Returns True if there are any forbidden substring Returns False if there aren't any forbidden substring ''' if self.forbidden_regexp_compiled is None: return False self.logger.debug("name_check: path %s" % req.path) self.logger.debug("name_check: self.forbidden_regexp %s" % self.forbidden_regexp) unquoted_path = unquote(req.path) match = self.forbidden_regexp_compiled.search(unquoted_path) return (match is not None) def __call__(self, env, start_response): req = Request(env) if self.check_character(req): return HTTPBadRequest( request=req, body=("Object/Container name contains forbidden chars from %s" % self.forbidden_chars))(env, start_response) elif self.check_length(req): return HTTPBadRequest( request=req, body=("Object/Container name longer than the allowed maximum " "%s" % self.maximum_length))(env, start_response) elif self.check_regexp(req): return HTTPBadRequest( request=req, body=("Object/Container name contains a forbidden substring " "from regular expression %s" % self.forbidden_regexp))(env, start_response) else: # Pass on to downstream WSGI component return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def name_check_filter(app): return NameCheckMiddleware(app, conf) return name_check_filter swift-1.13.1/swift/common/middleware/account_quotas.py0000664000175400017540000001211412323703611024227 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ ``account_quotas`` is a middleware which blocks write requests (PUT, POST) if a given account quota (in bytes) is exceeded while DELETE requests are still allowed. ``account_quotas`` uses the ``x-account-meta-quota-bytes`` metadata entry to store the quota. Write requests to this metadata entry are only permitted for resellers. There is no quota limit if ``x-account-meta-quota-bytes`` is not set. The ``account_quotas`` middleware should be added to the pipeline in your ``/etc/swift/proxy-server.conf`` file just after any auth middleware. For example:: [pipeline:main] pipeline = catch_errors cache tempauth account_quotas proxy-server [filter:account_quotas] use = egg:swift#account_quotas To set the quota on an account:: swift -A http://127.0.0.1:8080/auth/v1.0 -U account:reseller -K secret \ post -m quota-bytes:10000 Remove the quota:: swift -A http://127.0.0.1:8080/auth/v1.0 -U account:reseller -K secret \ post -m quota-bytes: The same limitations apply for the account quotas as for the container quotas. For example, when uploading an object without a content-length header the proxy server doesn't know the final size of the currently uploaded object and the upload will be allowed if the current account size is within the quota. Due to the eventual consistency further uploads might be possible until the account size has been updated. """ from swift.common.constraints import check_copy_from_header from swift.common.swob import HTTPForbidden, HTTPRequestEntityTooLarge, \ HTTPBadRequest, wsgify from swift.common.utils import register_swift_info from swift.proxy.controllers.base import get_account_info, get_object_info class AccountQuotaMiddleware(object): """Account quota middleware See above for a full description. """ def __init__(self, app, *args, **kwargs): self.app = app @wsgify def __call__(self, request): if request.method not in ("POST", "PUT", "COPY"): return self.app try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get( 'X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get( 'X-Remove-Account-Meta-Quota-Bytes') else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if request.method == "POST" or not obj: return self.app if request.method == 'COPY': copy_from = container + '/' + obj else: if 'x-copy-from' in request.headers: src_cont, src_obj = check_copy_from_header(request) copy_from = "%s/%s" % (src_cont, src_obj) else: copy_from = None content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app try: quota = int(account_info['meta'].get('quota-bytes', -1)) except ValueError: return self.app if quota < 0: return self.app if copy_from: path = '/' + ver + '/' + account + '/' + copy_from object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length if quota < new_size: return HTTPRequestEntityTooLarge() return self.app def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" register_swift_info('account_quotas') def account_quota_filter(app): return AccountQuotaMiddleware(app) return account_quota_filter swift-1.13.1/swift/common/middleware/memcache.py0000664000175400017540000000702112323703611022742 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os from ConfigParser import ConfigParser, NoSectionError, NoOptionError from swift.common.memcached import MemcacheRing class MemcacheMiddleware(object): """ Caching middleware that manages caching in swift. """ def __init__(self, app, conf): self.app = app self.memcache_servers = conf.get('memcache_servers') serialization_format = conf.get('memcache_serialization_support') try: # Originally, while we documented using memcache_max_connections # we only accepted max_connections max_conns = int(conf.get('memcache_max_connections', conf.get('max_connections', 0))) except ValueError: max_conns = 0 if (not self.memcache_servers or serialization_format is None or max_conns <= 0): path = os.path.join(conf.get('swift_dir', '/etc/swift'), 'memcache.conf') memcache_conf = ConfigParser() if memcache_conf.read(path): if not self.memcache_servers: try: self.memcache_servers = \ memcache_conf.get('memcache', 'memcache_servers') except (NoSectionError, NoOptionError): pass if serialization_format is None: try: serialization_format = \ memcache_conf.get('memcache', 'memcache_serialization_support') except (NoSectionError, NoOptionError): pass if max_conns <= 0: try: new_max_conns = \ memcache_conf.get('memcache', 'memcache_max_connections') max_conns = int(new_max_conns) except (NoSectionError, NoOptionError, ValueError): pass if not self.memcache_servers: self.memcache_servers = '127.0.0.1:11211' if max_conns <= 0: max_conns = 2 if serialization_format is None: serialization_format = 2 else: serialization_format = int(serialization_format) self.memcache = MemcacheRing( [s.strip() for s in self.memcache_servers.split(',') if s.strip()], allow_pickle=(serialization_format == 0), allow_unpickle=(serialization_format <= 1), max_conns=max_conns) def __call__(self, env, start_response): env['swift.cache'] = self.memcache return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def cache_filter(app): return MemcacheMiddleware(app, conf) return cache_filter swift-1.13.1/swift/common/middleware/tempurl.py0000664000175400017540000005045412323703611022700 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ TempURL Middleware Allows the creation of URLs to provide temporary access to objects. For example, a website may wish to provide a link to download a large object in Swift, but the Swift account has no public access. The website can generate a URL that will provide GET access for a limited time to the resource. When the web browser user clicks on the link, the browser will download the object directly from Swift, obviating the need for the website to act as a proxy for the request. If the user were to share the link with all his friends, or accidentally post it on a forum, etc. the direct access would be limited to the expiration time set when the website created the link. To create such temporary URLs, first an X-Account-Meta-Temp-URL-Key header must be set on the Swift account. Then, an HMAC-SHA1 (RFC 2104) signature is generated using the HTTP method to allow (GET or PUT), the Unix timestamp the access should be allowed until, the full path to the object, and the key set on the account. For example, here is code generating the signature for a GET for 60 seconds on /v1/AUTH_account/container/object:: import hmac from hashlib import sha1 from time import time method = 'GET' expires = int(time() + 60) path = '/v1/AUTH_account/container/object' key = 'mykey' hmac_body = '%s\\n%s\\n%s' % (method, expires, path) sig = hmac.new(key, hmac_body, sha1).hexdigest() Be certain to use the full path, from the /v1/ onward. Let's say the sig ends up equaling da39a3ee5e6b4b0d3255bfef95601890afd80709 and expires ends up 1323479485. Then, for example, the website could provide a link to:: https://swift-cluster.example.com/v1/AUTH_account/container/object? temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709& temp_url_expires=1323479485 Any alteration of the resource path or query arguments would result in 401 Unauthorized. Similary, a PUT where GET was the allowed method would 401. HEAD is allowed if GET or PUT is allowed. Using this in combination with browser form post translation middleware could also allow direct-from-browser uploads to specific locations in Swift. TempURL supports up to two keys, specified by X-Account-Meta-Temp-URL-Key and X-Account-Meta-Temp-URL-Key-2. Signatures are checked against both keys, if present. This is to allow for key rotation without invalidating all existing temporary URLs. With GET TempURLs, a Content-Disposition header will be set on the response so that browsers will interpret this as a file attachment to be saved. The filename chosen is based on the object name, but you can override this with a filename query parameter. Modifying the above example:: https://swift-cluster.example.com/v1/AUTH_account/container/object? temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709& temp_url_expires=1323479485&filename=My+Test+File.pdf If you do not want the object to be downloaded, you can cause "Content-Disposition: inline" to be set on the response by adding the "inline" parameter to the query string, like so:: https://swift-cluster.example.com/v1/AUTH_account/container/object? temp_url_sig=da39a3ee5e6b4b0d3255bfef95601890afd80709& temp_url_expires=1323479485&inline """ __all__ = ['TempURL', 'filter_factory', 'DEFAULT_INCOMING_REMOVE_HEADERS', 'DEFAULT_INCOMING_ALLOW_HEADERS', 'DEFAULT_OUTGOING_REMOVE_HEADERS', 'DEFAULT_OUTGOING_ALLOW_HEADERS'] from os.path import basename from time import time from urllib import urlencode from urlparse import parse_qs from swift.proxy.controllers.base import get_account_info from swift.common.swob import HeaderKeyDict, HTTPUnauthorized from swift.common.utils import split_path, get_valid_utf8_str, \ register_swift_info, get_hmac, streq_const_time #: Default headers to remove from incoming requests. Simply a whitespace #: delimited list of header names and names can optionally end with '*' to #: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of #: exceptions to these removals. DEFAULT_INCOMING_REMOVE_HEADERS = 'x-timestamp' #: Default headers as exceptions to DEFAULT_INCOMING_REMOVE_HEADERS. Simply a #: whitespace delimited list of header names and names can optionally end with #: '*' to indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS = '' #: Default headers to remove from outgoing responses. Simply a whitespace #: delimited list of header names and names can optionally end with '*' to #: indicate a prefix match. DEFAULT_OUTGOING_ALLOW_HEADERS is a list of #: exceptions to these removals. DEFAULT_OUTGOING_REMOVE_HEADERS = 'x-object-meta-*' #: Default headers as exceptions to DEFAULT_OUTGOING_REMOVE_HEADERS. Simply a #: whitespace delimited list of header names and names can optionally end with #: '*' to indicate a prefix match. DEFAULT_OUTGOING_ALLOW_HEADERS = 'x-object-meta-public-*' def get_tempurl_keys_from_metadata(meta): """ Extracts the tempurl keys from metadata. :param meta: account metadata :returns: list of keys found (possibly empty if no keys set) Example: meta = get_account_info(...)['meta'] keys = get_tempurl_keys_from_metadata(meta) """ return [get_valid_utf8_str(value) for key, value in meta.iteritems() if key.lower() in ('temp-url-key', 'temp-url-key-2')] class TempURL(object): """ WSGI Middleware to grant temporary URLs specific access to Swift resources. See the overview for more information. This middleware understands the following configuration settings:: incoming_remove_headers The headers to remove from incoming requests. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. incoming_allow_headers is a list of exceptions to these removals. Default: x-timestamp incoming_allow_headers The headers allowed as exceptions to incoming_remove_headers. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. Default: None outgoing_remove_headers The headers to remove from outgoing responses. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. outgoing_allow_headers is a list of exceptions to these removals. Default: x-object-meta-* outgoing_allow_headers The headers allowed as exceptions to outgoing_remove_headers. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. Default: x-object-meta-public-* The proxy logs created for any subrequests made will have swift.source set to "FP". :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf, methods=('GET', 'HEAD', 'PUT')): #: The next WSGI application/filter in the paste.deploy pipeline. self.app = app #: The filter configuration dict. self.conf = conf #: The methods allowed with Temp URLs. self.methods = methods headers = DEFAULT_INCOMING_REMOVE_HEADERS if 'incoming_remove_headers' in conf: headers = conf['incoming_remove_headers'] headers = \ ['HTTP_' + h.upper().replace('-', '_') for h in headers.split()] #: Headers to remove from incoming requests. Uppercase WSGI env style, #: like `HTTP_X_PRIVATE`. self.incoming_remove_headers = [h for h in headers if h[-1] != '*'] #: Header with match prefixes to remove from incoming requests. #: Uppercase WSGI env style, like `HTTP_X_SENSITIVE_*`. self.incoming_remove_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] headers = DEFAULT_INCOMING_ALLOW_HEADERS if 'incoming_allow_headers' in conf: headers = conf['incoming_allow_headers'] headers = \ ['HTTP_' + h.upper().replace('-', '_') for h in headers.split()] #: Headers to allow in incoming requests. Uppercase WSGI env style, #: like `HTTP_X_MATCHES_REMOVE_PREFIX_BUT_OKAY`. self.incoming_allow_headers = [h for h in headers if h[-1] != '*'] #: Header with match prefixes to allow in incoming requests. Uppercase #: WSGI env style, like `HTTP_X_MATCHES_REMOVE_PREFIX_BUT_OKAY_*`. self.incoming_allow_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] headers = DEFAULT_OUTGOING_REMOVE_HEADERS if 'outgoing_remove_headers' in conf: headers = conf['outgoing_remove_headers'] headers = [h.title() for h in headers.split()] #: Headers to remove from outgoing responses. Lowercase, like #: `x-account-meta-temp-url-key`. self.outgoing_remove_headers = [h for h in headers if h[-1] != '*'] #: Header with match prefixes to remove from outgoing responses. #: Lowercase, like `x-account-meta-private-*`. self.outgoing_remove_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] headers = DEFAULT_OUTGOING_ALLOW_HEADERS if 'outgoing_allow_headers' in conf: headers = conf['outgoing_allow_headers'] headers = [h.title() for h in headers.split()] #: Headers to allow in outgoing responses. Lowercase, like #: `x-matches-remove-prefix-but-okay`. self.outgoing_allow_headers = [h for h in headers if h[-1] != '*'] #: Header with match prefixes to allow in outgoing responses. #: Lowercase, like `x-matches-remove-prefix-but-okay-*`. self.outgoing_allow_headers_startswith = \ [h[:-1] for h in headers if h[-1] == '*'] #: HTTP user agent to use for subrequests. self.agent = '%(orig)s TempURL' def __call__(self, env, start_response): """ Main hook into the WSGI paste.deploy filter/app pipeline. :param env: The WSGI environment dict. :param start_response: The WSGI start_response hook. :returns: Response as per WSGI. """ if env['REQUEST_METHOD'] == 'OPTIONS': return self.app(env, start_response) info = self._get_temp_url_info(env) temp_url_sig, temp_url_expires, filename, inline_disposition = info if temp_url_sig is None and temp_url_expires is None: return self.app(env, start_response) if not temp_url_sig or not temp_url_expires: return self._invalid(env, start_response) account = self._get_account(env) if not account: return self._invalid(env, start_response) keys = self._get_keys(env, account) if not keys: return self._invalid(env, start_response) if env['REQUEST_METHOD'] == 'HEAD': hmac_vals = ( self._get_hmacs(env, temp_url_expires, keys) + self._get_hmacs(env, temp_url_expires, keys, request_method='GET') + self._get_hmacs(env, temp_url_expires, keys, request_method='PUT')) else: hmac_vals = self._get_hmacs(env, temp_url_expires, keys) # While it's true that any() will short-circuit, this doesn't affect # the timing-attack resistance since the only way this will # short-circuit is when a valid signature is passed in. is_valid_hmac = any(streq_const_time(temp_url_sig, hmac) for hmac in hmac_vals) if not is_valid_hmac: return self._invalid(env, start_response) self._clean_incoming_headers(env) env['swift.authorize'] = lambda req: None env['swift.authorize_override'] = True env['REMOTE_USER'] = '.wsgi.tempurl' qs = {'temp_url_sig': temp_url_sig, 'temp_url_expires': temp_url_expires} if filename: qs['filename'] = filename env['QUERY_STRING'] = urlencode(qs) def _start_response(status, headers, exc_info=None): headers = self._clean_outgoing_headers(headers) if env['REQUEST_METHOD'] == 'GET' and status[0] == '2': # figure out the right value for content-disposition # 1) use the value from the query string # 2) use the value from the object metadata # 3) use the object name (default) out_headers = [] existing_disposition = None for h, v in headers: if h.lower() != 'content-disposition': out_headers.append((h, v)) else: existing_disposition = v if inline_disposition: disposition_value = 'inline' elif filename: disposition_value = 'attachment; filename="%s"' % ( filename.replace('"', '\\"')) elif existing_disposition: disposition_value = existing_disposition else: name = basename(env['PATH_INFO'].rstrip('/')) disposition_value = 'attachment; filename="%s"' % ( name.replace('"', '\\"')) out_headers.append(('Content-Disposition', disposition_value)) headers = out_headers return start_response(status, headers, exc_info) return self.app(env, _start_response) def _get_account(self, env): """ Returns just the account for the request, if it's an object request and one of the configured methods; otherwise, None is returned. :param env: The WSGI environment for the request. :returns: Account str or None. """ if env['REQUEST_METHOD'] in self.methods: try: ver, acc, cont, obj = split_path(env['PATH_INFO'], 4, 4, True) except ValueError: return None if ver == 'v1' and obj.strip('/'): return acc def _get_temp_url_info(self, env): """ Returns the provided temporary URL parameters (sig, expires), if given and syntactically valid. Either sig or expires could be None if not provided. If provided, expires is also converted to an int if possible or 0 if not, and checked for expiration (returns 0 if expired). :param env: The WSGI environment for the request. :returns: (sig, expires, filename, inline) as described above. """ temp_url_sig = temp_url_expires = filename = inline = None qs = parse_qs(env.get('QUERY_STRING', ''), keep_blank_values=True) if 'temp_url_sig' in qs: temp_url_sig = qs['temp_url_sig'][0] if 'temp_url_expires' in qs: try: temp_url_expires = int(qs['temp_url_expires'][0]) except ValueError: temp_url_expires = 0 if temp_url_expires < time(): temp_url_expires = 0 if 'filename' in qs: filename = qs['filename'][0] if 'inline' in qs: inline = True return temp_url_sig, temp_url_expires, filename, inline def _get_keys(self, env, account): """ Returns the X-Account-Meta-Temp-URL-Key[-2] header values for the account, or an empty list if none is set. Returns 0, 1, or 2 elements depending on how many keys are set in the account's metadata. :param env: The WSGI environment for the request. :param account: Account str. :returns: [X-Account-Meta-Temp-URL-Key str value if set, X-Account-Meta-Temp-URL-Key-2 str value if set] """ account_info = get_account_info(env, self.app, swift_source='TU') return get_tempurl_keys_from_metadata(account_info['meta']) def _get_hmacs(self, env, expires, keys, request_method=None): """ :param env: The WSGI environment for the request. :param expires: Unix timestamp as an int for when the URL expires. :param keys: Key strings, from the X-Account-Meta-Temp-URL-Key[-2] of the account. :param request_method: Optional override of the request in the WSGI env. For example, if a HEAD does not match, you may wish to override with GET to still allow the HEAD. """ if not request_method: request_method = env['REQUEST_METHOD'] return [get_hmac( request_method, env['PATH_INFO'], expires, key) for key in keys] def _invalid(self, env, start_response): """ Performs the necessary steps to indicate a WSGI 401 Unauthorized response to the request. :param env: The WSGI environment for the request. :param start_response: The WSGI start_response hook. :returns: 401 response as per WSGI. """ if env['REQUEST_METHOD'] == 'HEAD': body = None else: body = '401 Unauthorized: Temp URL invalid\n' return HTTPUnauthorized(body=body)(env, start_response) def _clean_incoming_headers(self, env): """ Removes any headers from the WSGI environment as per the middleware configuration for incoming requests. :param env: The WSGI environment for the request. """ for h in env.keys(): remove = h in self.incoming_remove_headers if not remove: for p in self.incoming_remove_headers_startswith: if h.startswith(p): remove = True break if remove: if h in self.incoming_allow_headers: remove = False if remove: for p in self.incoming_allow_headers_startswith: if h.startswith(p): remove = False break if remove: del env[h] def _clean_outgoing_headers(self, headers): """ Removes any headers as per the middleware configuration for outgoing responses. :param headers: A WSGI start_response style list of headers, [('header1', 'value), ('header2', 'value), ...] :returns: The same headers list, but with some headers removed as per the middlware configuration for outgoing responses. """ headers = HeaderKeyDict(headers) for h in headers.keys(): remove = h in self.outgoing_remove_headers if not remove: for p in self.outgoing_remove_headers_startswith: if h.startswith(p): remove = True break if remove: if h in self.outgoing_allow_headers: remove = False if remove: for p in self.outgoing_allow_headers_startswith: if h.startswith(p): remove = False break if remove: del headers[h] return headers.items() def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) methods = conf.get('methods', 'GET HEAD PUT').split() register_swift_info('tempurl', methods=methods) return lambda app: TempURL(app, conf, methods=methods) swift-1.13.1/swift/common/middleware/tempauth.py0000664000175400017540000007152112323703611023035 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from time import time from traceback import format_exc from urllib import unquote from uuid import uuid4 from hashlib import sha1 import hmac import base64 from eventlet import Timeout from swift.common.swob import Response, Request from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \ HTTPUnauthorized from swift.common.request_helpers import get_sys_meta_prefix from swift.common.middleware.acl import ( clean_acl, parse_acl, referrer_allowed, acls_from_account_info) from swift.common.utils import cache_from_env, get_logger, \ split_path, config_true_value, register_swift_info from swift.proxy.controllers.base import get_account_info class TempAuth(object): """ Test authentication and authorization system. Add to your pipeline in proxy-server.conf, such as:: [pipeline:main] pipeline = catch_errors cache tempauth proxy-server Set account auto creation to true in proxy-server.conf:: [app:proxy-server] account_autocreate = true And add a tempauth filter section, such as:: [filter:tempauth] use = egg:swift#tempauth user_admin_admin = admin .admin .reseller_admin user_test_tester = testing .admin user_test2_tester2 = testing2 .admin user_test_tester3 = testing3 # To allow accounts/users with underscores you can base64 encode them. # Here is the account "under_score" and username "a_b" (note the lack # of padding equal signs): user64_dW5kZXJfc2NvcmU_YV9i = testing4 See the proxy-server.conf-sample for more information. Account ACLs: If a swift_owner issues a POST or PUT to the account, with the X-Account-Access-Control header set in the request, then this may allow certain types of access for additional users. * Read-Only: Users with read-only access can list containers in the account, list objects in any container, retrieve objects, and view unprivileged account/container/object metadata. * Read-Write: Users with read-write access can (in addition to the read-only privileges) create objects, overwrite existing objects, create new containers, and set unprivileged container/object metadata. * Admin: Users with admin access are swift_owners and can perform any action, including viewing/setting privileged metadata (e.g. changing account ACLs). To generate headers for setting an account ACL:: from swift.common.middleware.acl import format_acl acl_data = { 'admin': ['alice'], 'read-write': ['bob', 'carol'] } header_value = format_acl(version=2, acl_dict=acl_data) To generate a curl command line from the above:: token=... storage_url=... python -c ' from swift.common.middleware.acl import format_acl acl_data = { 'admin': ['alice'], 'read-write': ['bob', 'carol'] } headers = {'X-Account-Access-Control': format_acl(version=2, acl_dict=acl_data)} header_str = ' '.join(["-H '%s: %s'" % (k, v) for k, v in headers.items()]) print ('curl -D- -X POST -H "x-auth-token: $token" %s ' '$storage_url' % header_str) ' :param app: The next WSGI app in the pipeline :param conf: The dict of configuration values from the Paste config file """ def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(conf, log_route='tempauth') self.log_headers = config_true_value(conf.get('log_headers', 'f')) self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.logger.set_statsd_prefix('tempauth.%s' % ( self.reseller_prefix if self.reseller_prefix else 'NONE',)) self.auth_prefix = conf.get('auth_prefix', '/auth/') if not self.auth_prefix or not self.auth_prefix.strip('/'): self.logger.warning('Rewriting invalid auth prefix "%s" to ' '"/auth/" (Non-empty auth prefix path ' 'is required)' % self.auth_prefix) self.auth_prefix = '/auth/' if self.auth_prefix[0] != '/': self.auth_prefix = '/' + self.auth_prefix if self.auth_prefix[-1] != '/': self.auth_prefix += '/' self.token_life = int(conf.get('token_life', 86400)) self.allow_overrides = config_true_value( conf.get('allow_overrides', 't')) self.storage_url_scheme = conf.get('storage_url_scheme', 'default') self.users = {} for conf_key in conf: if conf_key.startswith('user_') or conf_key.startswith('user64_'): account, username = conf_key.split('_', 1)[1].split('_') if conf_key.startswith('user64_'): # Because trailing equal signs would screw up config file # parsing, we auto-pad with '=' chars. account += '=' * (len(account) % 4) account = base64.b64decode(account) username += '=' * (len(username) % 4) username = base64.b64decode(username) values = conf[conf_key].split() if not values: raise ValueError('%s has no key set' % conf_key) key = values.pop(0) if values and ('://' in values[-1] or '$HOST' in values[-1]): url = values.pop() else: url = '$HOST/v1/%s%s' % (self.reseller_prefix, account) self.users[account + ':' + username] = { 'key': key, 'url': url, 'groups': values} def __call__(self, env, start_response): """ Accepts a standard WSGI application call, authenticating the request and installing callback hooks for authorization and ACL header validation. For an authenticated request, REMOTE_USER will be set to a comma separated list of the user's groups. With a non-empty reseller prefix, acts as the definitive auth service for just tokens and accounts that begin with that prefix, but will deny requests outside this prefix if no other auth middleware overrides it. With an empty reseller prefix, acts as the definitive auth service only for tokens that validate to a non-empty set of groups. For all other requests, acts as the fallback auth service when no other auth middleware overrides it. Alternatively, if the request matches the self.auth_prefix, the request will be routed through the internal auth request handler (self.handle). This is to handle granting tokens, etc. """ if self.allow_overrides and env.get('swift.authorize_override', False): return self.app(env, start_response) if env.get('PATH_INFO', '').startswith(self.auth_prefix): return self.handle(env, start_response) s3 = env.get('HTTP_AUTHORIZATION') token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) if s3 or (token and token.startswith(self.reseller_prefix)): # Note: Empty reseller_prefix will match all tokens. groups = self.get_groups(env, token) if groups: user = groups and groups.split(',', 1)[0] or '' trans_id = env.get('swift.trans_id') self.logger.debug('User: %s uses token %s (trans_id %s)' % (user, 's3' if s3 else token, trans_id)) env['REMOTE_USER'] = groups env['swift.authorize'] = self.authorize env['swift.clean_acl'] = clean_acl if '.reseller_admin' in groups: env['reseller_request'] = True else: # Unauthorized token if self.reseller_prefix and not s3: # Because I know I'm the definitive auth for this token, I # can deny it outright. self.logger.increment('unauthorized') try: vrs, realm, rest = split_path(env['PATH_INFO'], 2, 3, True) except ValueError: realm = 'unknown' return HTTPUnauthorized(headers={ 'Www-Authenticate': 'Swift realm="%s"' % realm})( env, start_response) # Because I'm not certain if I'm the definitive auth for empty # reseller_prefixed tokens, I won't overwrite swift.authorize. elif 'swift.authorize' not in env: env['swift.authorize'] = self.denied_response else: if self.reseller_prefix: # With a non-empty reseller_prefix, I would like to be called # back for anonymous access to accounts I know I'm the # definitive auth for. try: version, rest = split_path(env.get('PATH_INFO', ''), 1, 2, True) except ValueError: version, rest = None, None self.logger.increment('errors') if rest and rest.startswith(self.reseller_prefix): # Handle anonymous access to accounts I'm the definitive # auth for. env['swift.authorize'] = self.authorize env['swift.clean_acl'] = clean_acl # Not my token, not my account, I can't authorize this request, # deny all is a good idea if not already set... elif 'swift.authorize' not in env: env['swift.authorize'] = self.denied_response # Because I'm not certain if I'm the definitive auth for empty # reseller_prefixed accounts, I won't overwrite swift.authorize. elif 'swift.authorize' not in env: env['swift.authorize'] = self.authorize env['swift.clean_acl'] = clean_acl return self.app(env, start_response) def _get_user_groups(self, account, account_user, account_id): """ :param account: example: test :param account_user: example: test:tester """ groups = [account, account_user] groups.extend(self.users[account_user]['groups']) if '.admin' in groups: groups.remove('.admin') groups.append(account_id) groups = ','.join(groups) return groups def get_groups(self, env, token): """ Get groups for the given token. :param env: The current WSGI environment dictionary. :param token: Token to validate and return a group string for. :returns: None if the token is invalid or a string containing a comma separated list of groups the authenticated user is a member of. The first group in the list is also considered a unique identifier for that user. """ groups = None memcache_client = cache_from_env(env) if not memcache_client: raise Exception('Memcache required') memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token) cached_auth_data = memcache_client.get(memcache_token_key) if cached_auth_data: expires, groups = cached_auth_data if expires < time(): groups = None if env.get('HTTP_AUTHORIZATION'): account_user, sign = \ env['HTTP_AUTHORIZATION'].split(' ')[1].rsplit(':', 1) if account_user not in self.users: return None account, user = account_user.split(':', 1) account_id = self.users[account_user]['url'].rsplit('/', 1)[-1] path = env['PATH_INFO'] env['PATH_INFO'] = path.replace(account_user, account_id, 1) msg = base64.urlsafe_b64decode(unquote(token)) key = self.users[account_user]['key'] s = base64.encodestring(hmac.new(key, msg, sha1).digest()).strip() if s != sign: return None groups = self._get_user_groups(account, account_user, account_id) return groups def account_acls(self, req): """ Return a dict of ACL data from the account server via get_account_info. Auth systems may define their own format, serialization, structure, and capabilities implemented in the ACL headers and persisted in the sysmeta data. However, auth systems are strongly encouraged to be interoperable with Tempauth. Account ACLs are set and retrieved via the header X-Account-Access-Control For header format and syntax, see: * :func:`swift.common.middleware.acl.parse_acl()` * :func:`swift.common.middleware.acl.format_acl()` """ info = get_account_info(req.environ, self.app, swift_source='TA') try: acls = acls_from_account_info(info) except ValueError as e1: self.logger.warn("Invalid ACL stored in metadata: %r" % e1) return None except NotImplementedError as e2: self.logger.warn("ACL version exceeds middleware version: %r" % e2) return None return acls def extract_acl_and_report_errors(self, req): """ Return a user-readable string indicating the errors in the input ACL, or None if there are no errors. """ acl_header = 'x-account-access-control' acl_data = req.headers.get(acl_header) result = parse_acl(version=2, data=acl_data) if result is None: return 'Syntax error in input (%r)' % acl_data tempauth_acl_keys = 'admin read-write read-only'.split() for key in result: # While it is possible to construct auth systems that collaborate # on ACLs, TempAuth is not such an auth system. At this point, # it thinks it is authoritative. if key not in tempauth_acl_keys: return 'Key %r not recognized' % key for key in tempauth_acl_keys: if key not in result: continue if not isinstance(result[key], list): return 'Value for key %r must be a list' % key for grantee in result[key]: if not isinstance(grantee, str): return 'Elements of %r list must be strings' % key # Everything looks fine, no errors found internal_hdr = get_sys_meta_prefix('account') + 'core-access-control' req.headers[internal_hdr] = req.headers.pop(acl_header) return None def authorize(self, req): """ Returns None if the request is authorized to continue or a standard WSGI response callable if not. """ try: _junk, account, container, obj = req.split_path(1, 4, True) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not account or not account.startswith(self.reseller_prefix): self.logger.debug("Account name: %s doesn't start with " "reseller_prefix: %s." % (account, self.reseller_prefix)) return self.denied_response(req) # At this point, TempAuth is convinced that it is authoritative. # If you are sending an ACL header, it must be syntactically valid # according to TempAuth's rules for ACL syntax. acl_data = req.headers.get('x-account-access-control') if acl_data is not None: error = self.extract_acl_and_report_errors(req) if error: msg = 'X-Account-Access-Control invalid: %s\n\nInput: %s\n' % ( error, acl_data) headers = [('Content-Type', 'text/plain; charset=UTF-8')] return HTTPBadRequest(request=req, headers=headers, body=msg) user_groups = (req.remote_user or '').split(',') account_user = user_groups[1] if len(user_groups) > 1 else None if '.reseller_admin' in user_groups and \ account != self.reseller_prefix and \ account[len(self.reseller_prefix)] != '.': req.environ['swift_owner'] = True self.logger.debug("User %s has reseller admin authorizing." % account_user) return None if account in user_groups and \ (req.method not in ('DELETE', 'PUT') or container): # If the user is admin for the account and is not trying to do an # account DELETE or PUT... req.environ['swift_owner'] = True self.logger.debug("User %s has admin authorizing." % account_user) return None if (req.environ.get('swift_sync_key') and (req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None)) and 'x-timestamp' in req.headers): self.logger.debug("Allow request with container sync-key: %s." % req.environ['swift_sync_key']) return None if req.method == 'OPTIONS': #allow OPTIONS requests to proceed as normal self.logger.debug("Allow OPTIONS request.") return None referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug("Allow authorizing %s via referer ACL." % req.referer) return None for user_group in user_groups: if user_group in groups: self.logger.debug("User %s allowed in ACL: %s authorizing." % (account_user, user_group)) return None # Check for access via X-Account-Access-Control acct_acls = self.account_acls(req) if acct_acls: # At least one account ACL is set in this account's sysmeta data, # so we should see whether this user is authorized by the ACLs. user_group_set = set(user_groups) if user_group_set.intersection(acct_acls['admin']): req.environ['swift_owner'] = True self.logger.debug('User %s allowed by X-Account-Access-Control' ' (admin)' % account_user) return None if (user_group_set.intersection(acct_acls['read-write']) and (container or req.method in ('GET', 'HEAD'))): # The RW ACL allows all operations to containers/objects, but # only GET/HEAD to accounts (and OPTIONS, above) self.logger.debug('User %s allowed by X-Account-Access-Control' ' (read-write)' % account_user) return None if (user_group_set.intersection(acct_acls['read-only']) and req.method in ('GET', 'HEAD')): self.logger.debug('User %s allowed by X-Account-Access-Control' ' (read-only)' % account_user) return None return self.denied_response(req) def denied_response(self, req): """ Returns a standard WSGI response callable with the status of 403 or 401 depending on whether the REMOTE_USER is set or not. """ if req.remote_user: self.logger.increment('forbidden') return HTTPForbidden(request=req) else: self.logger.increment('unauthorized') return HTTPUnauthorized(request=req) def handle(self, env, start_response): """ WSGI entry point for auth requests (ones that match the self.auth_prefix). Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ try: req = Request(env) if self.auth_prefix: req.path_info_pop() req.bytes_transferred = '-' req.client_disconnect = False if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token'] return self.handle_request(req)(env, start_response) except (Exception, Timeout): print "EXCEPTION IN handle: %s: %s" % (format_exc(), env) self.logger.increment('errors') start_response('500 Server Error', [('Content-Type', 'text/plain')]) return ['Internal server error.\n'] def handle_request(self, req): """ Entry point for auth requests (ones that match the self.auth_prefix). Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ req.start_time = time() handler = None try: version, account, user, _junk = req.split_path(1, 4, True) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if version in ('v1', 'v1.0', 'auth'): if req.method == 'GET': handler = self.handle_get_token if not handler: self.logger.increment('errors') req.response = HTTPBadRequest(request=req) else: req.response = handler(req) return req.response def handle_get_token(self, req): """ Handles the various `request for token and service end point(s)` calls. There are various formats to support the various auth servers in the past. Examples:: GET /v1//auth X-Auth-User: : or X-Storage-User: X-Auth-Key: or X-Storage-Pass: GET /auth X-Auth-User: : or X-Storage-User: : X-Auth-Key: or X-Storage-Pass: GET /v1.0 X-Auth-User: : or X-Storage-User: : X-Auth-Key: or X-Storage-Pass: On successful authentication, the response will have X-Auth-Token and X-Storage-Token set to the token to use with Swift and X-Storage-URL set to the URL to the default Swift cluster to use. :param req: The swob.Request to process. :returns: swob.Response, 2xx on success with data set as explained above. """ # Validate the request info try: pathsegs = split_path(req.path_info, 1, 3, True) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': account = pathsegs[1] user = req.headers.get('x-storage-user') if not user: user = req.headers.get('x-auth-user') if not user or ':' not in user: self.logger.increment('token_denied') return HTTPUnauthorized(request=req, headers= {'Www-Authenticate': 'Swift realm="%s"' % account}) account2, user = user.split(':', 1) if account != account2: self.logger.increment('token_denied') return HTTPUnauthorized(request=req, headers= {'Www-Authenticate': 'Swift realm="%s"' % account}) key = req.headers.get('x-storage-pass') if not key: key = req.headers.get('x-auth-key') elif pathsegs[0] in ('auth', 'v1.0'): user = req.headers.get('x-auth-user') if not user: user = req.headers.get('x-storage-user') if not user or ':' not in user: self.logger.increment('token_denied') return HTTPUnauthorized(request=req, headers= {'Www-Authenticate': 'Swift realm="unknown"'}) account, user = user.split(':', 1) key = req.headers.get('x-auth-key') if not key: key = req.headers.get('x-storage-pass') else: return HTTPBadRequest(request=req) if not all((account, user, key)): self.logger.increment('token_denied') realm = account or 'unknown' return HTTPUnauthorized(request=req, headers={'Www-Authenticate': 'Swift realm="%s"' % realm}) # Authenticate user account_user = account + ':' + user if account_user not in self.users: self.logger.increment('token_denied') return HTTPUnauthorized(request=req, headers= {'Www-Authenticate': 'Swift realm="%s"' % account}) if self.users[account_user]['key'] != key: self.logger.increment('token_denied') return HTTPUnauthorized(request=req, headers= {'Www-Authenticate': 'Swift realm="unknown"'}) account_id = self.users[account_user]['url'].rsplit('/', 1)[-1] # Get memcache client memcache_client = cache_from_env(req.environ) if not memcache_client: raise Exception('Memcache required') # See if a token already exists and hasn't expired token = None memcache_user_key = '%s/user/%s' % (self.reseller_prefix, account_user) candidate_token = memcache_client.get(memcache_user_key) if candidate_token: memcache_token_key = \ '%s/token/%s' % (self.reseller_prefix, candidate_token) cached_auth_data = memcache_client.get(memcache_token_key) if cached_auth_data: expires, old_groups = cached_auth_data old_groups = old_groups.split(',') new_groups = self._get_user_groups(account, account_user, account_id) if expires > time() and \ set(old_groups) == set(new_groups.split(',')): token = candidate_token # Create a new token if one didn't exist if not token: # Generate new token token = '%stk%s' % (self.reseller_prefix, uuid4().hex) expires = time() + self.token_life groups = self._get_user_groups(account, account_user, account_id) # Save token memcache_token_key = '%s/token/%s' % (self.reseller_prefix, token) memcache_client.set(memcache_token_key, (expires, groups), time=float(expires - time())) # Record the token with the user info for future use. memcache_user_key = \ '%s/user/%s' % (self.reseller_prefix, account_user) memcache_client.set(memcache_user_key, token, time=float(expires - time())) resp = Response(request=req, headers={ 'x-auth-token': token, 'x-storage-token': token}) url = self.users[account_user]['url'].replace('$HOST', resp.host_url) if self.storage_url_scheme != 'default': url = self.storage_url_scheme + ':' + url.split(':', 1)[1] resp.headers['x-storage-url'] = url return resp def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('tempauth', account_acls=True) def auth_filter(app): return TempAuth(app, conf) return auth_filter swift-1.13.1/swift/common/middleware/ratelimit.py0000664000175400017540000002700112323703611023172 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import time from swift import gettext_ as _ import eventlet from swift.common.utils import cache_from_env, get_logger, register_swift_info from swift.proxy.controllers.base import get_container_memcache_key, \ get_account_info from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request, Response def interpret_conf_limits(conf, name_prefix): conf_limits = [] for conf_key in conf: if conf_key.startswith(name_prefix): cont_size = int(conf_key[len(name_prefix):]) rate = float(conf[conf_key]) conf_limits.append((cont_size, rate)) conf_limits.sort() ratelimits = [] while conf_limits: cur_size, cur_rate = conf_limits.pop(0) if conf_limits: next_size, next_rate = conf_limits[0] slope = (float(next_rate) - float(cur_rate)) \ / (next_size - cur_size) def new_scope(cur_size, slope, cur_rate): # making new scope for variables return lambda x: (x - cur_size) * slope + cur_rate line_func = new_scope(cur_size, slope, cur_rate) else: line_func = lambda x: cur_rate ratelimits.append((cur_size, cur_rate, line_func)) return ratelimits def get_maxrate(ratelimits, size): """ Returns number of requests allowed per second for given size. """ last_func = None if size: size = int(size) for ratesize, rate, func in ratelimits: if size < ratesize: break last_func = func if last_func: return last_func(size) return None class MaxSleepTimeHitError(Exception): pass class RateLimitMiddleware(object): """ Rate limiting middleware Rate limits requests on both an Account and Container level. Limits are configurable. """ BLACK_LIST_SLEEP = 1 def __init__(self, app, conf, logger=None): self.app = app if logger: self.logger = logger else: self.logger = get_logger(conf, log_route='ratelimit') self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) self.log_sleep_time_seconds = \ float(conf.get('log_sleep_time_seconds', 0)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) self.rate_buffer_seconds = int(conf.get('rate_buffer_seconds', 5)) self.ratelimit_whitelist = \ [acc.strip() for acc in conf.get('account_whitelist', '').split(',') if acc.strip()] self.ratelimit_blacklist = \ [acc.strip() for acc in conf.get('account_blacklist', '').split(',') if acc.strip()] self.memcache_client = None self.container_ratelimits = interpret_conf_limits( conf, 'container_ratelimit_') self.container_listing_ratelimits = interpret_conf_limits( conf, 'container_listing_ratelimit_') def get_container_size(self, account_name, container_name): rv = 0 memcache_key = get_container_memcache_key(account_name, container_name) container_info = self.memcache_client.get(memcache_key) if isinstance(container_info, dict): rv = container_info.get( 'object_count', container_info.get('container_size', 0)) return rv def get_ratelimitable_key_tuples(self, req, account_name, container_name=None, obj_name=None): """ Returns a list of key (used in memcache), ratelimit tuples. Keys should be checked in order. :param req: swob request :param account_name: account name from path :param container_name: container name from path :param obj_name: object name from path """ keys = [] # COPYs are not limited if self.account_ratelimit and \ account_name and container_name and not obj_name and \ req.method in ('PUT', 'DELETE'): keys.append(("ratelimit/%s" % account_name, self.account_ratelimit)) if account_name and container_name and obj_name and \ req.method in ('PUT', 'DELETE', 'POST', 'COPY'): container_size = self.get_container_size( account_name, container_name) container_rate = get_maxrate( self.container_ratelimits, container_size) if container_rate: keys.append(( "ratelimit/%s/%s" % (account_name, container_name), container_rate)) if account_name and container_name and not obj_name and \ req.method == 'GET': container_size = self.get_container_size( account_name, container_name) container_rate = get_maxrate( self.container_listing_ratelimits, container_size) if container_rate: keys.append(( "ratelimit_listing/%s/%s" % (account_name, container_name), container_rate)) if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'): account_info = get_account_info(req.environ, self.app) account_global_ratelimit = \ account_info.get('sysmeta', {}).get('global-write-ratelimit') if account_global_ratelimit: try: account_global_ratelimit = float(account_global_ratelimit) if account_global_ratelimit > 0: keys.append(( "ratelimit/global-write/%s" % account_name, account_global_ratelimit)) except ValueError: pass return keys def _get_sleep_time(self, key, max_rate): ''' Returns the amount of time (a float in seconds) that the app should sleep. :param key: a memcache key :param max_rate: maximum rate allowed in requests per second :raises: MaxSleepTimeHitError if max sleep time is exceeded. ''' try: now_m = int(round(time.time() * self.clock_accuracy)) time_per_request_m = int(round(self.clock_accuracy / max_rate)) running_time_m = self.memcache_client.incr( key, delta=time_per_request_m) need_to_sleep_m = 0 if (now_m - running_time_m > self.rate_buffer_seconds * self.clock_accuracy): next_avail_time = int(now_m + time_per_request_m) self.memcache_client.set(key, str(next_avail_time), serialize=False) else: need_to_sleep_m = \ max(running_time_m - now_m - time_per_request_m, 0) max_sleep_m = self.max_sleep_time_seconds * self.clock_accuracy if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01: # treat as no-op decrement time self.memcache_client.decr(key, delta=time_per_request_m) raise MaxSleepTimeHitError( "Max Sleep Time Exceeded: %.2f" % (float(need_to_sleep_m) / self.clock_accuracy)) return float(need_to_sleep_m) / self.clock_accuracy except MemcacheConnectionError: return 0 def handle_ratelimit(self, req, account_name, container_name, obj_name): ''' Performs rate limiting and account white/black listing. Sleeps if necessary. If self.memcache_client is not set, immediately returns None. :param account_name: account name from path :param container_name: container name from path :param obj_name: object name from path ''' if not self.memcache_client: return None if account_name in self.ratelimit_blacklist: self.logger.error(_('Returning 497 because of blacklisting: %s'), account_name) eventlet.sleep(self.BLACK_LIST_SLEEP) return Response(status='497 Blacklisted', body='Your account has been blacklisted', request=req) if account_name in self.ratelimit_whitelist: return None for key, max_rate in self.get_ratelimitable_key_tuples( req, account_name, container_name=container_name, obj_name=obj_name): try: need_to_sleep = self._get_sleep_time(key, max_rate) if self.log_sleep_time_seconds and \ need_to_sleep > self.log_sleep_time_seconds: self.logger.warning( _("Ratelimit sleep log: %(sleep)s for " "%(account)s/%(container)s/%(object)s"), {'sleep': need_to_sleep, 'account': account_name, 'container': container_name, 'object': obj_name}) if need_to_sleep > 0: eventlet.sleep(need_to_sleep) except MaxSleepTimeHitError as e: self.logger.error( _('Returning 498 for %(meth)s to %(acc)s/%(cont)s/%(obj)s ' '. Ratelimit (Max Sleep) %(e)s'), {'meth': req.method, 'acc': account_name, 'cont': container_name, 'obj': obj_name, 'e': str(e)}) error_resp = Response(status='498 Rate Limited', body='Slow down', request=req) return error_resp return None def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env) if not self.memcache_client: self.logger.warning( _('Warning: Cannot ratelimit without a memcached client')) return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) ratelimit_resp = self.handle_ratelimit(req, account, container, obj) if ratelimit_resp is None: return self.app(env, start_response) else: return ratelimit_resp(env, start_response) def filter_factory(global_conf, **local_conf): """ paste.deploy app factory for creating WSGI proxy apps. """ conf = global_conf.copy() conf.update(local_conf) register_swift_info('ratelimit') def limit_filter(app): return RateLimitMiddleware(app, conf) return limit_filter swift-1.13.1/swift/common/middleware/container_sync.py0000664000175400017540000001211112323703611024212 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify from swift.common.utils import ( config_true_value, get_logger, register_swift_info, streq_const_time) from swift.proxy.controllers.base import get_container_info class ContainerSync(object): """ WSGI middleware that validates an incoming container sync request using the container-sync-realms.conf style of container sync. """ def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(conf, log_route='container_sync') self.realms_conf = ContainerSyncRealms( os.path.join( conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) self.allow_full_urls = config_true_value( conf.get('allow_full_urls', 'true')) @wsgify def __call__(self, req): if not self.allow_full_urls: sync_to = req.headers.get('x-container-sync-to') if sync_to and not sync_to.startswith('//'): raise HTTPBadRequest( body='Full URLs are not allowed for X-Container-Sync-To ' 'values. Only realm values of the format ' '//realm/cluster/account/container are allowed.\n', request=req) auth = req.headers.get('x-container-sync-auth') if auth: valid = False auth = auth.split() if len(auth) != 3: req.environ.setdefault('swift.log_info', []).append( 'cs:not-3-args') else: realm, nonce, sig = auth realm_key = self.realms_conf.key(realm) realm_key2 = self.realms_conf.key2(realm) if not realm_key: req.environ.setdefault('swift.log_info', []).append( 'cs:no-local-realm-key') else: info = get_container_info( req.environ, self.app, swift_source='CS') user_key = info.get('sync_key') if not user_key: req.environ.setdefault('swift.log_info', []).append( 'cs:no-local-user-key') else: expected = self.realms_conf.get_sig( req.method, req.path, req.headers.get('x-timestamp', '0'), nonce, realm_key, user_key) expected2 = self.realms_conf.get_sig( req.method, req.path, req.headers.get('x-timestamp', '0'), nonce, realm_key2, user_key) if realm_key2 else expected if not streq_const_time(sig, expected) and \ not streq_const_time(sig, expected2): req.environ.setdefault( 'swift.log_info', []).append('cs:invalid-sig') else: req.environ.setdefault( 'swift.log_info', []).append('cs:valid') valid = True if not valid: exc = HTTPUnauthorized( body='X-Container-Sync-Auth header not valid; ' 'contact cluster operator for support.', headers={'content-type': 'text/plain'}, request=req) exc.headers['www-authenticate'] = ' '.join([ 'SwiftContainerSync', exc.www_authenticate().split(None, 1)[1]]) raise exc else: req.environ['swift.authorize_override'] = True if req.path == '/info': # Ensure /info requests get the freshest results dct = {} for realm in self.realms_conf.realms(): clusters = self.realms_conf.clusters(realm) if clusters: dct[realm] = {'clusters': dict((c, {}) for c in clusters)} register_swift_info('container_sync', realms=dct) return self.app def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) register_swift_info('container_sync') def cache_filter(app): return ContainerSync(app, conf) return cache_filter swift-1.13.1/swift/common/middleware/proxy_logging.py0000664000175400017540000003135612323703614024102 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Logging middleware for the Swift proxy. This serves as both the default logging implementation and an example of how to plug in your own logging format/method. The logging format implemented below is as follows: client_ip remote_addr datetime request_method request_path protocol status_int referer user_agent auth_token bytes_recvd bytes_sent client_etag transaction_id headers request_time source log_info request_start_time request_end_time These values are space-separated, and each is url-encoded, so that they can be separated with a simple .split() * remote_addr is the contents of the REMOTE_ADDR environment variable, while client_ip is swift's best guess at the end-user IP, extracted variously from the X-Forwarded-For header, X-Cluster-Ip header, or the REMOTE_ADDR environment variable. * source (swift.source in the WSGI environment) indicates the code that generated the request, such as most middleware. (See below for more detail.) * log_info (swift.log_info in the WSGI environment) is for additional information that could prove quite useful, such as any x-delete-at value or other "behind the scenes" activity that might not otherwise be detectable from the plain log information. Code that wishes to add additional log information should use code like ``env.setdefault('swift.log_info', []).append(your_info)`` so as to not disturb others' log information. * Values that are missing (e.g. due to a header not being present) or zero are generally represented by a single hyphen ('-'). The proxy-logging can be used twice in the proxy server's pipeline when there is middleware installed that can return custom responses that don't follow the standard pipeline to the proxy server. For example, with staticweb, the middleware might intercept a request to /v1/AUTH_acc/cont/, make a subrequest to the proxy to retrieve /v1/AUTH_acc/cont/index.html and, in effect, respond to the client's original request using the 2nd request's body. In this instance the subrequest will be logged by the rightmost middleware (with a swift.source set) and the outgoing request (with body overridden) will be logged by leftmost middleware. Requests that follow the normal pipeline (use the same wsgi environment throughout) will not be double logged because an environment variable (swift.proxy_access_log_made) is checked/set when a log is made. All middleware making subrequests should take care to set swift.source when needed. With the doubled proxy logs, any consumer/processor of swift's proxy logs should look at the swift.source field, the rightmost log value, to decide if this is a middleware subrequest or not. A log processor calculating bandwidth usage will want to only sum up logs with no swift.source. """ import time from urllib import quote, unquote from swift.common.swob import Request from swift.common.utils import (get_logger, get_remote_client, get_valid_utf8_str, config_true_value, InputProxy, list_from_csv) from swift.common.constraints import MAX_HEADER_SIZE QUOTE_SAFE = '/:' class ProxyLoggingMiddleware(object): """ Middleware that logs Swift proxy requests in the swift log format. """ def __init__(self, app, conf, logger=None): self.app = app self.log_hdrs = config_true_value(conf.get( 'access_log_headers', conf.get('log_headers', 'no'))) log_hdrs_only = list_from_csv(conf.get( 'access_log_headers_only', '')) self.log_hdrs_only = [x.title() for x in log_hdrs_only] # The leading access_* check is in case someone assumes that # log_statsd_valid_http_methods behaves like the other log_statsd_* # settings. self.valid_methods = conf.get( 'access_log_statsd_valid_http_methods', conf.get('log_statsd_valid_http_methods', 'GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS')) self.valid_methods = [m.strip().upper() for m in self.valid_methods.split(',') if m.strip()] access_log_conf = {} for key in ('log_facility', 'log_name', 'log_level', 'log_udp_host', 'log_udp_port', 'log_statsd_host', 'log_statsd_port', 'log_statsd_default_sample_rate', 'log_statsd_sample_rate_factor', 'log_statsd_metric_prefix'): value = conf.get('access_' + key, conf.get(key, None)) if value: access_log_conf[key] = value self.access_logger = logger or get_logger(access_log_conf, log_route='proxy-access') self.access_logger.set_statsd_prefix('proxy-server') self.reveal_sensitive_prefix = int(conf.get('reveal_sensitive_prefix', MAX_HEADER_SIZE)) def method_from_req(self, req): return req.environ.get('swift.orig_req_method', req.method) def req_already_logged(self, env): return env.get('swift.proxy_access_log_made') def mark_req_logged(self, env): env['swift.proxy_access_log_made'] = True def obscure_sensitive(self, value): if value and len(value) > self.reveal_sensitive_prefix: return value[:self.reveal_sensitive_prefix] + '...' return value def log_request(self, req, status_int, bytes_received, bytes_sent, start_time, end_time): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param start_time: timestamp request started :param end_time: timestamp request completed """ req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path), QUOTE_SAFE) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: if self.log_hdrs_only: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items() if k in self.log_hdrs_only) else: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) end_gmtime_str = time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime(end_time)) duration_time_str = "%.4f" % (end_time - start_time) start_time_str = "%.9f" % start_time end_time_str = "%.9f" % end_time self.access_logger.info(' '.join( quote(str(x) if x else '-', QUOTE_SAFE) for x in ( get_remote_client(req), req.remote_addr, end_gmtime_str, method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, self.obscure_sensitive(req.headers.get('x-auth-token')), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, duration_time_str, req.environ.get('swift.source'), ','.join(req.environ.get('swift.log_info') or ''), start_time_str, end_time_str ))) # Log timing and bytes-transfered data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent) def statsd_metric_name(self, req, status_int, method): if req.path.startswith('/v1/'): try: stat_type = [None, 'account', 'container', 'object'][req.path.strip('/').count('/')] except IndexError: stat_type = 'object' else: stat_type = req.environ.get('swift.source') if stat_type is None: return None stat_method = method if method in self.valid_methods \ else 'BAD_METHOD' return '.'.join((stat_type, stat_method, str(status_int))) def __call__(self, env, start_response): if self.req_already_logged(env): return self.app(env, start_response) self.mark_req_logged(env) start_response_args = [None] input_proxy = InputProxy(env['wsgi.input']) env['wsgi.input'] = input_proxy start_time = time.time() def my_start_response(status, headers, exc_info=None): start_response_args[0] = (status, list(headers), exc_info) def status_int_for_logging(client_disconnect=False, start_status=None): # log disconnected clients as '499' status code if client_disconnect or input_proxy.client_disconnect: ret_status_int = 499 elif start_status is None: ret_status_int = int( start_response_args[0][0].split(' ', 1)[0]) else: ret_status_int = start_status return ret_status_int def iter_response(iterable): iterator = iter(iterable) try: chunk = iterator.next() while not chunk: chunk = iterator.next() except StopIteration: chunk = '' for h, v in start_response_args[0][1]: if h.lower() in ('content-length', 'transfer-encoding'): break else: if not chunk: start_response_args[0][1].append(('content-length', '0')) elif isinstance(iterable, list): start_response_args[0][1].append( ('content-length', str(sum(len(i) for i in iterable)))) start_response(*start_response_args[0]) req = Request(env) # Log timing information for time-to-first-byte (GET requests only) method = self.method_from_req(req) if method == 'GET': status_int = status_int_for_logging() metric_name = self.statsd_metric_name(req, status_int, method) if metric_name: self.access_logger.timing_since( metric_name + '.first-byte.timing', start_time) bytes_sent = 0 client_disconnect = False try: while chunk: bytes_sent += len(chunk) yield chunk chunk = iterator.next() except GeneratorExit: # generator was closed before we finished client_disconnect = True raise finally: status_int = status_int_for_logging(client_disconnect) self.log_request( req, status_int, input_proxy.bytes_received, bytes_sent, start_time, time.time()) try: iterable = self.app(env, my_start_response) except Exception: req = Request(env) status_int = status_int_for_logging(start_status=500) self.log_request( req, status_int, input_proxy.bytes_received, 0, start_time, time.time()) raise else: return iter_response(iterable) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def proxy_logger(app): return ProxyLoggingMiddleware(app, conf) return proxy_logger swift-1.13.1/swift/common/middleware/slo.py0000664000175400017540000010301012323703611021770 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Middleware that will provide Static Large Object (SLO) support. This feature is very similar to Dynamic Large Object (DLO) support in that it allows the user to upload many objects concurrently and afterwards download them as a single object. It is different in that it does not rely on eventually consistent container listings to do so. Instead, a user defined manifest of the object segments is used. ---------------------- Uploading the Manifest ---------------------- After the user has uploaded the objects to be concatenated a manifest is uploaded. The request must be a PUT with the query parameter:: ?multipart-manifest=put The body of this request will be an ordered list of files in json data format. The data to be supplied for each segment is:: path: the path to the segment (not including account) /container/object_name etag: the etag given back when the segment was PUT size_bytes: the size of the segment in bytes The format of the list will be:: json: [{"path": "/cont/object", "etag": "etagoftheobjectsegment", "size_bytes": 1048576}, ...] The number of object segments is limited to a configurable amount, default 1000. Each segment, except for the final one, must be at least 1 megabyte (configurable). On upload, the middleware will head every segment passed in and verify the size and etag of each. If any of the objects do not match (not found, size/etag mismatch, below minimum size) then the user will receive a 4xx error response. If everything does match, the user will receive a 2xx response and the SLO object is ready for downloading. Behind the scenes, on success, a json manifest generated from the user input is sent to object servers with an extra "X-Static-Large-Object: True" header and a modified Content-Type. The parameter: swift_bytes=$total_size will be appended to the existing Content-Type, where total_size is the sum of all the included segments' size_bytes. This extra parameter will be hidden from the user. Manifest files can reference objects in separate containers, which will improve concurrent upload speed. Objects can be referenced by multiple manifests. The segments of a SLO manifest can even be other SLO manifests. Treat them as any other object i.e., use the Etag and Content-Length given on the PUT of the sub-SLO in the manifest to the parent SLO. ------------------------- Retrieving a Large Object ------------------------- A GET request to the manifest object will return the concatenation of the objects from the manifest much like DLO. If any of the segments from the manifest are not found or their Etag/Content Length no longer match the connection will drop. In this case a 409 Conflict will be logged in the proxy logs and the user will receive incomplete results. The headers from this GET or HEAD request will return the metadata attached to the manifest object itself with some exceptions:: Content-Length: the total size of the SLO (the sum of the sizes of the segments in the manifest) X-Static-Large-Object: True Etag: the etag of the SLO (generated the same way as DLO) A GET request with the query parameter:: ?multipart-manifest=get Will return the actual manifest file itself. This is generated json and does not match the data sent from the original multipart-manifest=put. This call's main purpose is for debugging. When the manifest object is uploaded you are more or less guaranteed that every segment in the manifest exists and matched the specifications. However, there is nothing that prevents the user from breaking the SLO download by deleting/replacing a segment referenced in the manifest. It is left to the user to use caution in handling the segments. ----------------------- Deleting a Large Object ----------------------- A DELETE request will just delete the manifest object itself. A DELETE with a query parameter:: ?multipart-manifest=delete will delete all the segments referenced in the manifest and then the manifest itself. The failure response will be similar to the bulk delete middleware. ------------------------ Modifying a Large Object ------------------------ PUTs / POSTs will work as expected, PUTs will just overwrite the manifest object for example. ------------------ Container Listings ------------------ In a container listing the size listed for SLO manifest objects will be the total_size of the concatenated segments in the manifest. The overall X-Container-Bytes-Used for the container (and subsequently for the account) will not reflect total_size of the manifest but the actual size of the json data stored. The reason for this somewhat confusing discrepancy is we want the container listing to reflect the size of the manifest object when it is downloaded. We do not, however, want to count the bytes-used twice (for both the manifest and the segments it's referring to) in the container and account metadata which can be used for stats purposes. """ from cStringIO import StringIO from datetime import datetime import mimetypes import re from hashlib import md5 from swift.common.exceptions import ListingIterError from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \ HTTPUnauthorized, HTTPRequestedRangeNotSatisfiable, Response from swift.common.utils import json, get_logger, config_true_value, \ get_valid_utf8_str, override_bytes_from_content_type, split_path, \ register_swift_info, RateLimitedIterator, quote from swift.common.request_helpers import SegmentedIterable, \ closing_if_possible, close_if_possible from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS from swift.common.http import HTTP_NOT_FOUND, HTTP_UNAUTHORIZED, is_success from swift.common.wsgi import WSGIContext, make_subrequest from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk DEFAULT_MIN_SEGMENT_SIZE = 1024 * 1024 # 1 MiB DEFAULT_MAX_MANIFEST_SEGMENTS = 1000 DEFAULT_MAX_MANIFEST_SIZE = 1024 * 1024 * 2 # 2 MiB def parse_input(raw_data): """ Given a request will parse the body and return a list of dictionaries :raises: HTTPException on parse errors :returns: a list of dictionaries on success """ try: parsed_data = json.loads(raw_data) except ValueError: raise HTTPBadRequest("Manifest must be valid json.") req_keys = set(['path', 'etag', 'size_bytes']) try: for seg_dict in parsed_data: if (set(seg_dict) != req_keys or '/' not in seg_dict['path'].lstrip('/')): raise HTTPBadRequest('Invalid SLO Manifest File') except (AttributeError, TypeError): raise HTTPBadRequest('Invalid SLO Manifest File') return parsed_data class SloPutContext(WSGIContext): def __init__(self, slo, slo_etag): super(SloPutContext, self).__init__(slo.app) self.slo_etag = '"' + slo_etag.hexdigest() + '"' def handle_slo_put(self, req, start_response): app_resp = self._app_call(req.environ) for i in xrange(len(self._response_headers)): if self._response_headers[i][0].lower() == 'etag': self._response_headers[i] = ('Etag', self.slo_etag) break start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp class SloGetContext(WSGIContext): max_slo_recursion_depth = 10 def __init__(self, slo): self.slo = slo self.first_byte = None self.last_byte = None super(SloGetContext, self).__init__(slo.app) def _fetch_sub_slo_segments(self, req, version, acc, con, obj): """ Fetch the submanifest, parse it, and return it. Raise exception on failures. """ sub_req = make_subrequest( req.environ, path='/'.join(['', version, acc, con, obj]), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO') sub_resp = sub_req.get_response(self.slo.app) if not is_success(sub_resp.status_int): raise ListingIterError( 'ERROR: while fetching %s, GET of submanifest %s ' 'failed with status %d' % (req.path, sub_req.path, sub_resp.status_int)) try: with closing_if_possible(sub_resp.app_iter): return json.loads(''.join(sub_resp.app_iter)) except ValueError as err: raise ListingIterError( 'ERROR: while fetching %s, JSON-decoding of submanifest %s ' 'failed with %s' % (req.path, sub_req.path, err)) def _segment_listing_iterator(self, req, version, account, segments, recursion_depth=1): for seg_dict in segments: if config_true_value(seg_dict.get('sub_slo')): override_bytes_from_content_type(seg_dict, logger=self.slo.logger) # We handle the range stuff here so that we can be smart about # skipping unused submanifests. For example, if our first segment is a # submanifest referencing 50 MiB total, but self.first_byte falls in # the 51st MiB, then we can avoid fetching the first submanifest. # # If we were to make SegmentedIterable handle all the range # calculations, we would be unable to make this optimization. total_length = sum(int(seg['bytes']) for seg in segments) if self.first_byte is None: self.first_byte = 0 if self.last_byte is None: self.last_byte = total_length - 1 for seg_dict in segments: seg_length = int(seg_dict['bytes']) if self.first_byte >= seg_length: # don't need any bytes from this segment self.first_byte = max(self.first_byte - seg_length, -1) self.last_byte = max(self.last_byte - seg_length, -1) continue if self.last_byte < 0: # no bytes are needed from this or any future segment break if config_true_value(seg_dict.get('sub_slo')): # do this check here so that we can avoid fetching this last # manifest before raising the exception if recursion_depth >= self.max_slo_recursion_depth: raise ListingIterError("Max recursion depth exceeded") sub_path = get_valid_utf8_str(seg_dict['name']) sub_cont, sub_obj = split_path(sub_path, 2, 2, True) sub_segments = self._fetch_sub_slo_segments( req, version, account, sub_cont, sub_obj) for sub_seg_dict, sb, eb in self._segment_listing_iterator( req, version, account, sub_segments, recursion_depth=recursion_depth + 1): yield sub_seg_dict, sb, eb else: if isinstance(seg_dict['name'], unicode): seg_dict['name'] = seg_dict['name'].encode("utf-8") seg_length = int(seg_dict['bytes']) yield (seg_dict, (None if self.first_byte <= 0 else self.first_byte), (None if self.last_byte >= seg_length - 1 else self.last_byte)) self.first_byte = max(self.first_byte - seg_length, -1) self.last_byte = max(self.last_byte - seg_length, -1) def _need_to_refetch_manifest(self, req): """ Just because a response shows that an object is a SLO manifest does not mean that response's body contains the entire SLO manifest. If it doesn't, we need to make a second request to actually get the whole thing. Note: this assumes that X-Static-Large-Object has already been found. """ if req.method == 'HEAD': return True response_status = int(self._response_status[:3]) # These are based on etag, and the SLO's etag is almost certainly not # the manifest object's etag. Still, it's highly likely that the # submitted If-None-Match won't match the manifest object's etag, so # we can avoid re-fetching the manifest if we got a successful # response. if ((req.if_match or req.if_none_match) and not is_success(response_status)): return True if req.range and response_status in (206, 416): content_range = '' for header, value in self._response_headers: if header.lower() == 'content-range': content_range = value break # e.g. Content-Range: bytes 0-14289/14290 match = re.match('bytes (\d+)-(\d+)/(\d+)$', content_range) if not match: # Malformed or missing, so we don't know what we got. return True first_byte, last_byte, length = [int(x) for x in match.groups()] # If and only if we actually got back the full manifest body, then # we can avoid re-fetching the object. got_everything = (first_byte == 0 and last_byte == length - 1) return not got_everything return False def handle_slo_get_or_head(self, req, start_response): """ Takes a request and a start_response callable and does the normal WSGI thing with them. Returns an iterator suitable for sending up the WSGI chain. :param req: swob.Request object; is a GET or HEAD request aimed at what may be a static large object manifest (or may not). :param start_response: WSGI start_response callable """ resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest for header, value in self._response_headers: if (header.lower() == 'x-static-large-object' and config_true_value(value)): break else: # Not a static large object manifest. Just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter # Handle pass-through request for the manifest itself if req.params.get('multipart-manifest') == 'get': new_headers = [] for header, value in self._response_headers: if header.lower() == 'content-type': new_headers.append(('Content-Type', 'application/json; charset=utf-8')) else: new_headers.append((header, value)) self._response_headers = new_headers start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter if self._need_to_refetch_manifest(req): req.environ['swift.non_client_disconnect'] = True close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO') resp_iter = self._app_call(get_req.environ) # Any Content-Range from a manifest is almost certainly wrong for the # full large object. resp_headers = [(h, v) for h, v in self._response_headers if not h.lower() == 'content-range'] response = self.get_or_head_response( req, resp_headers, resp_iter) return response(req.environ, start_response) def get_or_head_response(self, req, resp_headers, resp_iter): resp_body = ''.join(resp_iter) try: segments = json.loads(resp_body) except ValueError: segments = [] etag = md5() content_length = 0 for seg_dict in segments: etag.update(seg_dict['hash']) if config_true_value(seg_dict.get('sub_slo')): override_bytes_from_content_type( seg_dict, logger=self.slo.logger) content_length += int(seg_dict['bytes']) response_headers = [(h, v) for h, v in resp_headers if h.lower() not in ('etag', 'content-length')] response_headers.append(('Content-Length', str(content_length))) response_headers.append(('Etag', '"%s"' % etag.hexdigest())) if req.method == 'HEAD': return self._manifest_head_response(req, response_headers) else: return self._manifest_get_response( req, content_length, response_headers, segments) def _manifest_head_response(self, req, response_headers): return HTTPOk(request=req, headers=response_headers, body='', conditional_response=True) def _manifest_get_response(self, req, content_length, response_headers, segments): self.first_byte, self.last_byte = None, None if req.range: byteranges = req.range.ranges_for_length(content_length) if len(byteranges) == 0: return HTTPRequestedRangeNotSatisfiable(request=req) elif len(byteranges) == 1: self.first_byte, self.last_byte = byteranges[0] # For some reason, swob.Range.ranges_for_length adds 1 to the # last byte's position. self.last_byte -= 1 else: req.range = None ver, account, _junk = req.split_path(3, 3, rest_with_last=True) plain_listing_iter = self._segment_listing_iterator( req, ver, account, segments) ratelimited_listing_iter = RateLimitedIterator( plain_listing_iter, self.slo.rate_limit_segments_per_sec, limit_after=self.slo.rate_limit_after_segment) # self._segment_listing_iterator gives us 3-tuples of (segment dict, # start byte, end byte), but SegmentedIterable wants (obj path, etag, # size, start byte, end byte), so we clean that up here segment_listing_iter = ( ("/{ver}/{acc}/{conobj}".format( ver=ver, acc=account, conobj=seg_dict['name'].lstrip('/')), seg_dict['hash'], int(seg_dict['bytes']), start_byte, end_byte) for seg_dict, start_byte, end_byte in ratelimited_listing_iter) response = Response(request=req, content_length=content_length, headers=response_headers, conditional_response=True, app_iter=SegmentedIterable( req, self.slo.app, segment_listing_iter, name=req.path, logger=self.slo.logger, ua_suffix="SLO MultipartGET", swift_source="SLO", max_get_time=self.slo.max_get_time)) if req.range: response.headers.pop('Etag') return response class StaticLargeObject(object): """ StaticLargeObject Middleware See above for a full description. The proxy logs created for any subrequests made will have swift.source set to "SLO". :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf, min_segment_size=DEFAULT_MIN_SEGMENT_SIZE, max_manifest_segments=DEFAULT_MAX_MANIFEST_SEGMENTS, max_manifest_size=DEFAULT_MAX_MANIFEST_SIZE): self.conf = conf self.app = app self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = max_manifest_segments self.max_manifest_size = max_manifest_size self.min_segment_size = min_segment_size self.max_get_time = int(self.conf.get('max_get_time', 86400)) self.rate_limit_after_segment = int(self.conf.get( 'rate_limit_after_segment', '10')) self.rate_limit_segments_per_sec = int(self.conf.get( 'rate_limit_segments_per_sec', '0')) self.bulk_deleter = Bulk(app, {}, logger=self.logger) def handle_multipart_get_or_head(self, req, start_response): """ Handles the GET or HEAD of a SLO manifest. The response body (only on GET, of course) will consist of the concatenation of the segments. :params req: a swob.Request with a path referencing an object :raises: HttpException on errors """ return SloGetContext(self).handle_slo_get_or_head(req, start_response) def copy_hook(self, inner_hook): def slo_hook(source_req, source_resp, sink_req): x_slo = source_resp.headers.get('X-Static-Large-Object') if (config_true_value(x_slo) and source_req.params.get('multipart-manifest') != 'get'): source_resp = SloGetContext(self).get_or_head_response( source_req, source_resp.headers.items(), source_resp.app_iter) return inner_hook(source_req, source_resp, sink_req) return slo_hook def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. Heads every object in manifest to check if is valid and if so will save a manifest generated from the user input. Uses WSGIContext to call self and start_response and returns a WSGI iterator. :params req: a swob.Request with an obj in path :raises: HttpException on errors """ try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(req.environ, start_response) if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge( "Manifest File > %d bytes" % self.max_manifest_size) if req.headers.get('X-Copy-From'): raise HTTPMethodNotAllowed( 'Multipart Manifest PUTs cannot be COPY requests') if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) parsed_data = parse_input(req.body_file.read(self.max_manifest_size)) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: raise HTTPRequestEntityTooLarge( 'Number of segments must be <= %d' % self.max_manifest_segments) total_size = 0 out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] slo_etag = md5() for index, seg_dict in enumerate(parsed_data): obj_name = seg_dict['path'] if isinstance(obj_name, unicode): obj_name = obj_name.encode('utf-8') obj_path = '/'.join(['', vrs, account, obj_name.lstrip('/')]) try: seg_size = int(seg_dict['size_bytes']) except (ValueError, TypeError): raise HTTPBadRequest('Invalid Manifest File') if seg_size < self.min_segment_size and \ (index == 0 or index < len(parsed_data) - 1): raise HTTPBadRequest( 'Each segment, except the last, must be at least ' '%d bytes.' % self.min_segment_size) new_env = req.environ.copy() new_env['PATH_INFO'] = obj_path new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SLO' del(new_env['wsgi.input']) del(new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self) if head_seg_resp.is_success: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_name), 'Size Mismatch']) if seg_dict['etag'] == head_seg_resp.etag: slo_etag.update(seg_dict['etag']) else: problem_segments.append([quote(obj_name), 'Etag Mismatch']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: # shouldn't happen last_modified = datetime.now() last_modified_formatted = \ last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f') seg_data = {'name': '/' + seg_dict['path'].lstrip('/'), 'bytes': seg_size, 'hash': seg_dict['etag'], 'content_type': head_seg_resp.content_type, 'last_modified': last_modified_formatted} if config_true_value( head_seg_resp.headers.get('X-Static-Large-Object')): seg_data['sub_slo'] = True data_for_storage.append(seg_data) else: problem_segments.append([quote(obj_name), head_seg_resp.status]) if problem_segments: resp_body = get_response_body( out_content_type, {}, problem_segments) raise HTTPBadRequest(resp_body, content_type=out_content_type) env = req.environ if not env.get('CONTENT_TYPE'): guessed_type, _junk = mimetypes.guess_type(req.path_info) env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream' env['swift.content_type_overridden'] = True env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) slo_put_context = SloPutContext(self, slo_etag) return slo_put_context.handle_slo_put(req, start_response) def get_segments_to_delete_iter(self, req): """ A generator function to be used to delete all the segments and sub-segments referenced in a manifest. :params req: a swob.Request with an SLO manifest in path :raises HTTPPreconditionFailed: on invalid UTF8 in request path :raises HTTPBadRequest: on too many buffered sub segments and on invalid SLO manifest path """ if not check_utf8(req.path_info): raise HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') vrs, account, container, obj = req.split_path(4, 4, True) segments = [{ 'sub_slo': True, 'name': ('/%s/%s' % (container, obj)).decode('utf-8')}] while segments: if len(segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest( 'Too many buffered slo segments to delete.') seg_data = segments.pop(0) if seg_data.get('sub_slo'): try: segments.extend( self.get_slo_segments(seg_data['name'], req)) except HTTPException as err: # allow bulk delete response to report errors seg_data['error'] = {'code': err.status_int, 'message': err.body} # add manifest back to be deleted after segments seg_data['sub_slo'] = False segments.append(seg_data) else: seg_data['name'] = seg_data['name'].encode('utf-8') yield seg_data def get_slo_segments(self, obj_name, req): """ Performs a swob.Request and returns the SLO manifest's segments. :raises HTTPServerError: on unable to load obj_name or on unable to load the SLO manifest data. :raises HTTPBadRequest: on not an SLO manifest :raises HTTPNotFound: on SLO manifest not found :returns: SLO manifest's segments """ vrs, account, _junk = req.split_path(2, 3, True) new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' del(new_env['wsgi.input']) new_env['QUERY_STRING'] = 'multipart-manifest=get' new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartDELETE' % new_env.get('HTTP_USER_AGENT') new_env['swift.source'] = 'SLO' new_env['PATH_INFO'] = ( '/%s/%s/%s' % (vrs, account, obj_name.lstrip('/')) ).encode('utf-8') resp = Request.blank('', new_env).get_response(self.app) if resp.is_success: if config_true_value(resp.headers.get('X-Static-Large-Object')): try: return json.loads(resp.body) except ValueError: raise HTTPServerError('Unable to load SLO manifest') else: raise HTTPBadRequest('Not an SLO manifest') elif resp.status_int == HTTP_NOT_FOUND: raise HTTPNotFound('SLO manifest not found') elif resp.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized('401 Unauthorized') else: raise HTTPServerError('Unable to load SLO manifest or segment.') def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. :params req: a swob.Request with an obj in path :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter """ resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: resp.content_type = out_content_type resp.app_iter = self.bulk_deleter.handle_delete_iter( req, objs_to_delete=self.get_segments_to_delete_iter(req), user_agent='MultipartDELETE', swift_source='SLO', out_content_type=out_content_type) return resp def __call__(self, env, start_response): """ WSGI entry point """ req = Request(env) try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) # install our COPY-callback hook env['swift.copy_hook'] = self.copy_hook( env.get('swift.copy_hook', lambda src_req, src_resp, sink_req: src_resp)) try: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req, start_response) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req)(env, start_response) if req.method == 'GET' or req.method == 'HEAD': return self.handle_multipart_get_or_head(req, start_response) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') except HTTPException as err_resp: return err_resp(env, start_response) return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) max_manifest_segments = int(conf.get('max_manifest_segments', DEFAULT_MAX_MANIFEST_SEGMENTS)) max_manifest_size = int(conf.get('max_manifest_size', DEFAULT_MAX_MANIFEST_SIZE)) min_segment_size = int(conf.get('min_segment_size', DEFAULT_MIN_SEGMENT_SIZE)) register_swift_info('slo', max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, min_segment_size=min_segment_size) def slo_filter(app): return StaticLargeObject( app, conf, max_manifest_segments=max_manifest_segments, max_manifest_size=max_manifest_size, min_segment_size=min_segment_size) return slo_filter swift-1.13.1/swift/common/middleware/formpost.py0000664000175400017540000004467012323703611023064 0ustar jenkinsjenkins00000000000000# Copyright (c) 2011 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ FormPost Middleware Translates a browser form post into a regular Swift object PUT. The format of the form is::

The is the URL to the Swift desination, such as:: https://swift-cluster.example.com/v1/AUTH_account/container/object_prefix The name of each file uploaded will be appended to the given. So, you can upload directly to the root of container with a url like:: https://swift-cluster.example.com/v1/AUTH_account/container/ Optionally, you can include an object prefix to better separate different users' uploads, such as:: https://swift-cluster.example.com/v1/AUTH_account/container/object_prefix Note the form method must be POST and the enctype must be set as "multipart/form-data". The redirect attribute is the URL to redirect the browser to after the upload completes. This is an optional parameter. If you are uploading the form via an XMLHttpRequest the redirect should not be included. The URL will have status and message query parameters added to it, indicating the HTTP status code for the upload (2xx is success) and a possible message for further information if there was an error (such as "max_file_size exceeded"). The max_file_size attribute must be included and indicates the largest single file upload that can be done, in bytes. The max_file_count attribute must be included and indicates the maximum number of files that can be uploaded with the form. Include additional ```` attributes if desired. The expires attribute is the Unix timestamp before which the form must be submitted before it is invalidated. The signature attribute is the HMAC-SHA1 signature of the form. Here is sample code for computing the signature:: import hmac from hashlib import sha1 from time import time path = '/v1/account/container/object_prefix' redirect = 'https://srv.com/some-page' # set to '' if redirect not in form max_file_size = 104857600 max_file_count = 10 expires = int(time() + 600) key = 'mykey' hmac_body = '%s\\n%s\\n%s\\n%s\\n%s' % (path, redirect, max_file_size, max_file_count, expires) signature = hmac.new(key, hmac_body, sha1).hexdigest() The key is the value of the X-Account-Meta-Temp-URL-Key header on the account. Be certain to use the full path, from the /v1/ onward. The command line tool ``swift-form-signature`` may be used (mostly just when testing) to compute expires and signature. Also note that the file attributes must be after the other attributes in order to be processed correctly. If attributes come after the file, they won't be sent with the subrequest (there is no way to parse all the attributes on the server-side without reading the whole thing into memory -- to service many requests, some with large files, there just isn't enough memory on the server, so attributes following the file are simply ignored). """ __all__ = ['FormPost', 'filter_factory', 'READ_CHUNK_SIZE', 'MAX_VALUE_LENGTH'] import hmac import re import rfc822 from hashlib import sha1 from time import time from urllib import quote from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata from swift.common.utils import streq_const_time, register_swift_info from swift.common.wsgi import make_pre_authed_env from swift.common.swob import HTTPUnauthorized from swift.proxy.controllers.base import get_account_info #: The size of data to read from the form at any given time. READ_CHUNK_SIZE = 4096 #: The maximum size of any attribute's value. Any additional data will be #: truncated. MAX_VALUE_LENGTH = 4096 #: Regular expression to match form attributes. ATTRIBUTES_RE = re.compile(r'(\w+)=(".*?"|[^";]+)(; ?|$)') class FormInvalid(Exception): pass class FormUnauthorized(Exception): pass def _parse_attrs(header): """ Given the value of a header like: Content-Disposition: form-data; name="somefile"; filename="test.html" Return data like ("form-data", {"name": "somefile", "filename": "test.html"}) :param header: Value of a header (the part after the ': '). :returns: (value name, dict) of the attribute data parsed (see above). """ attributes = {} attrs = '' if '; ' in header: header, attrs = header.split('; ', 1) m = True while m: m = ATTRIBUTES_RE.match(attrs) if m: attrs = attrs[len(m.group(0)):] attributes[m.group(1)] = m.group(2).strip('"') return header, attributes class _IterRequestsFileLikeObject(object): def __init__(self, wsgi_input, boundary, input_buffer): self.no_more_data_for_this_file = False self.no_more_files = False self.wsgi_input = wsgi_input self.boundary = boundary self.input_buffer = input_buffer def read(self, length=None): if not length: length = READ_CHUNK_SIZE if self.no_more_data_for_this_file: return '' # read enough data to know whether we're going to run # into a boundary in next [length] bytes if len(self.input_buffer) < length + len(self.boundary) + 2: to_read = length + len(self.boundary) + 2 while to_read > 0: chunk = self.wsgi_input.read(to_read) to_read -= len(chunk) self.input_buffer += chunk if not chunk: self.no_more_files = True break boundary_pos = self.input_buffer.find(self.boundary) # boundary does not exist in the next (length) bytes if boundary_pos == -1 or boundary_pos > length: ret = self.input_buffer[:length] self.input_buffer = self.input_buffer[length:] # if it does, just return data up to the boundary else: ret, self.input_buffer = self.input_buffer.split(self.boundary, 1) self.no_more_files = self.input_buffer.startswith('--') self.no_more_data_for_this_file = True self.input_buffer = self.input_buffer[2:] return ret def readline(self): if self.no_more_data_for_this_file: return '' boundary_pos = newline_pos = -1 while newline_pos < 0 and boundary_pos < 0: chunk = self.wsgi_input.read(READ_CHUNK_SIZE) self.input_buffer += chunk newline_pos = self.input_buffer.find('\r\n') boundary_pos = self.input_buffer.find(self.boundary) if not chunk: self.no_more_files = True break # found a newline if newline_pos >= 0 and \ (boundary_pos < 0 or newline_pos < boundary_pos): # Use self.read to ensure any logic there happens... ret = '' to_read = newline_pos + 2 while to_read > 0: chunk = self.read(to_read) # Should never happen since we're reading from input_buffer, # but just for completeness... if not chunk: break to_read -= len(chunk) ret += chunk return ret else: # no newlines, just return up to next boundary return self.read(len(self.input_buffer)) def _iter_requests(wsgi_input, boundary): """ Given a multi-part mime encoded input file object and boundary, yield file-like objects for each part. :param wsgi_input: The file-like object to read from. :param boundary: The mime boundary to separate new file-like objects on. :returns: A generator of file-like objects for each part. """ boundary = '--' + boundary if wsgi_input.readline().strip() != boundary: raise FormInvalid('invalid starting boundary') boundary = '\r\n' + boundary input_buffer = '' done = False while not done: it = _IterRequestsFileLikeObject(wsgi_input, boundary, input_buffer) yield it done = it.no_more_files input_buffer = it.input_buffer class _CappedFileLikeObject(object): """ A file-like object wrapping another file-like object that raises an EOFError if the amount of data read exceeds a given max_file_size. :param fp: The file-like object to wrap. :param max_file_size: The maximum bytes to read before raising an EOFError. """ def __init__(self, fp, max_file_size): self.fp = fp self.max_file_size = max_file_size self.amount_read = 0 def read(self, size=None): ret = self.fp.read(size) self.amount_read += len(ret) if self.amount_read > self.max_file_size: raise EOFError('max_file_size exceeded') return ret def readline(self): ret = self.fp.readline() self.amount_read += len(ret) if self.amount_read > self.max_file_size: raise EOFError('max_file_size exceeded') return ret class FormPost(object): """ FormPost Middleware See above for a full description. The proxy logs created for any subrequests made will have swift.source set to "FP". :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf): #: The next WSGI application/filter in the paste.deploy pipeline. self.app = app #: The filter configuration dict. self.conf = conf def __call__(self, env, start_response): """ Main hook into the WSGI paste.deploy filter/app pipeline. :param env: The WSGI environment dict. :param start_response: The WSGI start_response hook. :returns: Response as per WSGI. """ if env['REQUEST_METHOD'] == 'POST': try: content_type, attrs = \ _parse_attrs(env.get('CONTENT_TYPE') or '') if content_type == 'multipart/form-data' and \ 'boundary' in attrs: http_user_agent = "%s FormPost" % ( env.get('HTTP_USER_AGENT', '')) env['HTTP_USER_AGENT'] = http_user_agent.strip() status, headers, body = self._translate_form( env, attrs['boundary']) start_response(status, headers) return body except (FormInvalid, EOFError) as err: body = 'FormPost: %s' % err start_response( '400 Bad Request', (('Content-Type', 'text/plain'), ('Content-Length', str(len(body))))) return [body] except FormUnauthorized as err: message = 'FormPost: %s' % str(err).title() return HTTPUnauthorized(body=message)( env, start_response) return self.app(env, start_response) def _translate_form(self, env, boundary): """ Translates the form data into subrequests and issues a response. :param env: The WSGI environment dict. :param boundary: The MIME type boundary to look for. :returns: status_line, headers_list, body """ keys = self._get_keys(env) status = message = '' attributes = {} subheaders = [] file_count = 0 for fp in _iter_requests(env['wsgi.input'], boundary): hdrs = rfc822.Message(fp, 0) disp, attrs = \ _parse_attrs(hdrs.getheader('Content-Disposition', '')) if disp == 'form-data' and attrs.get('filename'): file_count += 1 try: if file_count > int(attributes.get('max_file_count') or 0): status = '400 Bad Request' message = 'max file count exceeded' break except ValueError: raise FormInvalid('max_file_count not an integer') attributes['filename'] = attrs['filename'] or 'filename' if 'content-type' not in attributes and 'content-type' in hdrs: attributes['content-type'] = \ hdrs['Content-Type'] or 'application/octet-stream' status, subheaders, message = \ self._perform_subrequest(env, attributes, fp, keys) if status[:1] != '2': break else: data = '' mxln = MAX_VALUE_LENGTH while mxln: chunk = fp.read(mxln) if not chunk: break mxln -= len(chunk) data += chunk while fp.read(READ_CHUNK_SIZE): pass if 'name' in attrs: attributes[attrs['name'].lower()] = data.rstrip('\r\n--') if not status: status = '400 Bad Request' message = 'no files to process' headers = [(k, v) for k, v in subheaders if k.lower().startswith('access-control')] redirect = attributes.get('redirect') if not redirect: body = status if message: body = status + '\r\nFormPost: ' + message.title() headers.extend([('Content-Type', 'text/plain'), ('Content-Length', len(body))]) return status, headers, body status = status.split(' ', 1)[0] if '?' in redirect: redirect += '&' else: redirect += '?' redirect += 'status=%s&message=%s' % (quote(status), quote(message)) body = '

' \ 'Click to continue...

' % redirect headers.extend( [('Location', redirect), ('Content-Length', str(len(body)))]) return '303 See Other', headers, body def _perform_subrequest(self, orig_env, attributes, fp, keys): """ Performs the subrequest and returns the response. :param orig_env: The WSGI environment dict; will only be used to form a new env for the subrequest. :param attributes: dict of the attributes of the form so far. :param fp: The file-like object containing the request body. :param keys: The account keys to validate the signature with. :returns: (status_line, headers_list, message) """ if not keys: raise FormUnauthorized('invalid signature') try: max_file_size = int(attributes.get('max_file_size') or 0) except ValueError: raise FormInvalid('max_file_size not an integer') subenv = make_pre_authed_env(orig_env, 'PUT', agent=None, swift_source='FP') if 'QUERY_STRING' in subenv: del subenv['QUERY_STRING'] subenv['HTTP_TRANSFER_ENCODING'] = 'chunked' subenv['wsgi.input'] = _CappedFileLikeObject(fp, max_file_size) if subenv['PATH_INFO'][-1] != '/' and \ subenv['PATH_INFO'].count('/') < 4: subenv['PATH_INFO'] += '/' subenv['PATH_INFO'] += attributes['filename'] or 'filename' if 'content-type' in attributes: subenv['CONTENT_TYPE'] = \ attributes['content-type'] or 'application/octet-stream' elif 'CONTENT_TYPE' in subenv: del subenv['CONTENT_TYPE'] try: if int(attributes.get('expires') or 0) < time(): raise FormUnauthorized('form expired') except ValueError: raise FormInvalid('expired not an integer') hmac_body = '%s\n%s\n%s\n%s\n%s' % ( orig_env['PATH_INFO'], attributes.get('redirect') or '', attributes.get('max_file_size') or '0', attributes.get('max_file_count') or '0', attributes.get('expires') or '0') has_valid_sig = False for key in keys: sig = hmac.new(key, hmac_body, sha1).hexdigest() if streq_const_time(sig, (attributes.get('signature') or 'invalid')): has_valid_sig = True if not has_valid_sig: raise FormUnauthorized('invalid signature') substatus = [None] subheaders = [None] def _start_response(status, headers, exc_info=None): substatus[0] = status subheaders[0] = headers i = iter(self.app(subenv, _start_response)) try: i.next() except StopIteration: pass return substatus[0], subheaders[0], '' def _get_keys(self, env): """ Fetch the tempurl keys for the account. Also validate that the request path indicates a valid container; if not, no keys will be returned. :param env: The WSGI environment for the request. :returns: list of tempurl keys """ parts = env['PATH_INFO'].split('/', 4) if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \ not parts[3]: return [] account_info = get_account_info(env, self.app, swift_source='FP') return get_tempurl_keys_from_metadata(account_info['meta']) def filter_factory(global_conf, **local_conf): """Returns the WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('formpost') return lambda app: FormPost(app, conf) swift-1.13.1/swift/common/middleware/gatekeeper.py0000664000175400017540000000766012323703611023325 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ The ``gatekeeper`` middleware imposes restrictions on the headers that may be included with requests and responses. Request headers are filtered to remove headers that should never be generated by a client. Similarly, response headers are filtered to remove private headers that should never be passed to a client. The ``gatekeeper`` middleware must always be present in the proxy server wsgi pipeline. It should be configured close to the start of the pipeline specified in ``/etc/swift/proxy-server.conf``, immediately after catch_errors and before any other middleware. It is essential that it is configured ahead of all middlewares using system metadata in order that they function correctly. If ``gatekeeper`` middleware is not configured in the pipeline then it will be automatically inserted close to the start of the pipeline by the proxy server. """ from swift.common.swob import Request from swift.common.utils import get_logger from swift.common.request_helpers import remove_items, get_sys_meta_prefix import re #: A list of python regular expressions that will be used to #: match against inbound request headers. Matching headers will #: be removed from the request. # Exclude headers starting with a sysmeta prefix. # If adding to this list, note that these are regex patterns, # so use a trailing $ to constrain to an exact header match # rather than prefix match. inbound_exclusions = [get_sys_meta_prefix('account'), get_sys_meta_prefix('container'), get_sys_meta_prefix('object'), 'x-backend'] # 'x-object-sysmeta' is reserved in anticipation of future support # for system metadata being applied to objects #: A list of python regular expressions that will be used to #: match against outbound response headers. Matching headers will #: be removed from the response. outbound_exclusions = inbound_exclusions def make_exclusion_test(exclusions): expr = '|'.join(exclusions) test = re.compile(expr, re.IGNORECASE) return test.match class GatekeeperMiddleware(object): def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='gatekeeper') self.inbound_condition = make_exclusion_test(inbound_exclusions) self.outbound_condition = make_exclusion_test(outbound_exclusions) def __call__(self, env, start_response): req = Request(env) removed = remove_items(req.headers, self.inbound_condition) if removed: self.logger.debug('removed request headers: %s' % removed) def gatekeeper_response(status, response_headers, exc_info=None): removed = filter( lambda h: self.outbound_condition(h[0]), response_headers) if removed: self.logger.debug('removed response headers: %s' % removed) new_headers = filter( lambda h: not self.outbound_condition(h[0]), response_headers) return start_response(status, new_headers, exc_info) return start_response(status, response_headers, exc_info) return self.app(env, gatekeeper_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def gatekeeper_filter(app): return GatekeeperMiddleware(app, conf) return gatekeeper_filter swift-1.13.1/swift/common/middleware/list_endpoints.py0000664000175400017540000001263112323703611024241 0ustar jenkinsjenkins00000000000000# Copyright (c) 2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ List endpoints for an object, account or container. This middleware makes it possible to integrate swift with software that relies on data locality information to avoid network overhead, such as Hadoop. Answers requests of the form:: /endpoints/{account}/{container}/{object} /endpoints/{account}/{container} /endpoints/{account} with a JSON-encoded list of endpoints of the form:: http://{server}:{port}/{dev}/{part}/{acc}/{cont}/{obj} http://{server}:{port}/{dev}/{part}/{acc}/{cont} http://{server}:{port}/{dev}/{part}/{acc} correspondingly, e.g.:: http://10.1.1.1:6000/sda1/2/a/c2/o1 http://10.1.1.1:6000/sda1/2/a/c2 http://10.1.1.1:6000/sda1/2/a The '/endpoints/' path is customizable ('list_endpoints_path' configuration parameter). Intended for consumption by third-party services living inside the cluster (as the endpoints make sense only inside the cluster behind the firewall); potentially written in a different language. This is why it's provided as a REST API and not just a Python API: to avoid requiring clients to write their own ring parsers in their languages, and to avoid the necessity to distribute the ring file to clients and keep it up-to-date. Note that the call is not authenticated, which means that a proxy with this middleware enabled should not be open to an untrusted environment (everyone can query the locality data using this middleware). """ from urllib import quote, unquote from swift.common.ring import Ring from swift.common.utils import json, get_logger, split_path from swift.common.swob import Request, Response from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed class ListEndpointsMiddleware(object): """ List endpoints for an object, account or container. See above for a full description. Uses configuration parameter `swift_dir` (default `/etc/swift`). :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf): self.app = app self.logger = get_logger(conf, log_route='endpoints') swift_dir = conf.get('swift_dir', '/etc/swift') self.account_ring = Ring(swift_dir, ring_name='account') self.container_ring = Ring(swift_dir, ring_name='container') self.object_ring = Ring(swift_dir, ring_name='object') self.endpoints_path = conf.get('list_endpoints_path', '/endpoints/') if not self.endpoints_path.endswith('/'): self.endpoints_path += '/' def __call__(self, env, start_response): request = Request(env) if not request.path.startswith(self.endpoints_path): return self.app(env, start_response) if request.method != 'GET': return HTTPMethodNotAllowed( req=request, headers={"Allow": "GET"})(env, start_response) try: clean_path = request.path[len(self.endpoints_path) - 1:] account, container, obj = \ split_path(clean_path, 1, 3, True) except ValueError: return HTTPBadRequest('No account specified')(env, start_response) if account is not None: account = unquote(account) if container is not None: container = unquote(container) if obj is not None: obj = unquote(obj) if obj is not None: partition, nodes = self.object_ring.get_nodes( account, container, obj) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}/{obj}' elif container is not None: partition, nodes = self.container_ring.get_nodes( account, container) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}/{container}' else: partition, nodes = self.account_ring.get_nodes( account) endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \ '{account}' endpoints = [] for node in nodes: endpoint = endpoint_template.format( ip=node['ip'], port=node['port'], device=node['device'], partition=partition, account=quote(account), container=quote(container or ''), obj=quote(obj or '')) endpoints.append(endpoint) return Response(json.dumps(endpoints), content_type='application/json')(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def list_endpoints_filter(app): return ListEndpointsMiddleware(app, conf) return list_endpoints_filter swift-1.13.1/swift/common/middleware/recon.py0000664000175400017540000003454012323703611022314 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import errno import os from swift import gettext_ as _ from swift import __version__ as swiftver from swift.common.swob import Request, Response from swift.common.utils import get_logger, config_true_value, json from swift.common.constraints import check_mount from resource import getpagesize from hashlib import md5 class ReconMiddleware(object): """ Recon middleware used for monitoring. /recon/load|mem|async... will return various system metrics. Needs to be added to the pipeline and requires a filter declaration in the object-server.conf: [filter:recon] use = egg:swift#recon recon_cache_path = /var/cache/swift """ def __init__(self, app, conf, *args, **kwargs): self.app = app self.devices = conf.get('devices', '/srv/node') swift_dir = conf.get('swift_dir', '/etc/swift') self.logger = get_logger(conf, log_route='recon') self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.object_recon_cache = os.path.join(self.recon_cache_path, 'object.recon') self.container_recon_cache = os.path.join(self.recon_cache_path, 'container.recon') self.account_recon_cache = os.path.join(self.recon_cache_path, 'account.recon') self.account_ring_path = os.path.join(swift_dir, 'account.ring.gz') self.container_ring_path = os.path.join(swift_dir, 'container.ring.gz') self.object_ring_path = os.path.join(swift_dir, 'object.ring.gz') self.rings = [self.account_ring_path, self.container_ring_path, self.object_ring_path] self.mount_check = config_true_value(conf.get('mount_check', 'true')) def _from_recon_cache(self, cache_keys, cache_file, openr=open): """retrieve values from a recon cache file :params cache_keys: list of cache items to retrieve :params cache_file: cache file to retrieve items from. :params openr: open to use [for unittests] :return: dict of cache items and their values or none if not found """ try: with openr(cache_file, 'r') as f: recondata = json.load(f) return dict((key, recondata.get(key)) for key in cache_keys) except IOError: self.logger.exception(_('Error reading recon cache file')) except ValueError: self.logger.exception(_('Error parsing recon cache file')) except Exception: self.logger.exception(_('Error retrieving recon data')) return dict((key, None) for key in cache_keys) def get_version(self): """get swift version""" verinfo = {'version': swiftver} return verinfo def get_mounted(self, openr=open): """get ALL mounted fs from /proc/mounts""" mounts = [] with openr('/proc/mounts', 'r') as procmounts: for line in procmounts: mount = {} mount['device'], mount['path'], opt1, opt2, opt3, \ opt4 = line.rstrip().split() mounts.append(mount) return mounts def get_load(self, openr=open): """get info from /proc/loadavg""" loadavg = {} with openr('/proc/loadavg', 'r') as f: onemin, fivemin, ftmin, tasks, procs = f.read().rstrip().split() loadavg['1m'] = float(onemin) loadavg['5m'] = float(fivemin) loadavg['15m'] = float(ftmin) loadavg['tasks'] = tasks loadavg['processes'] = int(procs) return loadavg def get_mem(self, openr=open): """get info from /proc/meminfo""" meminfo = {} with openr('/proc/meminfo', 'r') as memlines: for i in memlines: entry = i.rstrip().split(":") meminfo[entry[0]] = entry[1].strip() return meminfo def get_async_info(self): """get # of async pendings""" return self._from_recon_cache(['async_pending'], self.object_recon_cache) def get_replication_info(self, recon_type): """get replication info""" if recon_type == 'account': return self._from_recon_cache(['replication_time', 'replication_stats', 'replication_last'], self.account_recon_cache) elif recon_type == 'container': return self._from_recon_cache(['replication_time', 'replication_stats', 'replication_last'], self.container_recon_cache) elif recon_type == 'object': return self._from_recon_cache(['object_replication_time', 'object_replication_last'], self.object_recon_cache) else: return None def get_device_info(self): """get devices""" try: return {self.devices: os.listdir(self.devices)} except Exception: self.logger.exception(_('Error listing devices')) return {self.devices: None} def get_updater_info(self, recon_type): """get updater info""" if recon_type == 'container': return self._from_recon_cache(['container_updater_sweep'], self.container_recon_cache) elif recon_type == 'object': return self._from_recon_cache(['object_updater_sweep'], self.object_recon_cache) else: return None def get_expirer_info(self, recon_type): """get expirer info""" if recon_type == 'object': return self._from_recon_cache(['object_expiration_pass', 'expired_last_pass'], self.object_recon_cache) def get_auditor_info(self, recon_type): """get auditor info""" if recon_type == 'account': return self._from_recon_cache(['account_audits_passed', 'account_auditor_pass_completed', 'account_audits_since', 'account_audits_failed'], self.account_recon_cache) elif recon_type == 'container': return self._from_recon_cache(['container_audits_passed', 'container_auditor_pass_completed', 'container_audits_since', 'container_audits_failed'], self.container_recon_cache) elif recon_type == 'object': return self._from_recon_cache(['object_auditor_stats_ALL', 'object_auditor_stats_ZBF'], self.object_recon_cache) else: return None def get_unmounted(self): """list unmounted (failed?) devices""" mountlist = [] for entry in os.listdir(self.devices): try: mounted = check_mount(self.devices, entry) except OSError as err: mounted = str(err) mpoint = {'device': entry, 'mounted': mounted} if mpoint['mounted'] is not True: mountlist.append(mpoint) return mountlist def get_diskusage(self): """get disk utilization statistics""" devices = [] for entry in os.listdir(self.devices): try: mounted = check_mount(self.devices, entry) except OSError as err: devices.append({'device': entry, 'mounted': str(err), 'size': '', 'used': '', 'avail': ''}) continue if mounted: path = os.path.join(self.devices, entry) disk = os.statvfs(path) capacity = disk.f_bsize * disk.f_blocks available = disk.f_bsize * disk.f_bavail used = disk.f_bsize * (disk.f_blocks - disk.f_bavail) devices.append({'device': entry, 'mounted': True, 'size': capacity, 'used': used, 'avail': available}) else: devices.append({'device': entry, 'mounted': False, 'size': '', 'used': '', 'avail': ''}) return devices def get_ring_md5(self, openr=open): """get all ring md5sum's""" sums = {} for ringfile in self.rings: md5sum = md5() if os.path.exists(ringfile): try: with openr(ringfile, 'rb') as f: block = f.read(4096) while block: md5sum.update(block) block = f.read(4096) sums[ringfile] = md5sum.hexdigest() except IOError as err: sums[ringfile] = None if err.errno != errno.ENOENT: self.logger.exception(_('Error reading ringfile')) return sums def get_quarantine_count(self): """get obj/container/account quarantine counts""" qcounts = {"objects": 0, "containers": 0, "accounts": 0} qdir = "quarantined" for device in os.listdir(self.devices): for qtype in qcounts: qtgt = os.path.join(self.devices, device, qdir, qtype) if os.path.exists(qtgt): linkcount = os.lstat(qtgt).st_nlink if linkcount > 2: qcounts[qtype] += linkcount - 2 return qcounts def get_socket_info(self, openr=open): """ get info from /proc/net/sockstat and sockstat6 Note: The mem value is actually kernel pages, but we return bytes allocated based on the systems page size. """ sockstat = {} try: with openr('/proc/net/sockstat', 'r') as proc_sockstat: for entry in proc_sockstat: if entry.startswith("TCP: inuse"): tcpstats = entry.split() sockstat['tcp_in_use'] = int(tcpstats[2]) sockstat['orphan'] = int(tcpstats[4]) sockstat['time_wait'] = int(tcpstats[6]) sockstat['tcp_mem_allocated_bytes'] = \ int(tcpstats[10]) * getpagesize() except IOError as e: if e.errno != errno.ENOENT: raise try: with openr('/proc/net/sockstat6', 'r') as proc_sockstat6: for entry in proc_sockstat6: if entry.startswith("TCP6: inuse"): sockstat['tcp6_in_use'] = int(entry.split()[2]) except IOError as e: if e.errno != errno.ENOENT: raise return sockstat def GET(self, req): root, rcheck, rtype = req.split_path(1, 3, True) all_rtypes = ['account', 'container', 'object'] if rcheck == "mem": content = self.get_mem() elif rcheck == "load": content = self.get_load() elif rcheck == "async": content = self.get_async_info() elif rcheck == 'replication' and rtype in all_rtypes: content = self.get_replication_info(rtype) elif rcheck == 'replication' and rtype is None: #handle old style object replication requests content = self.get_replication_info('object') elif rcheck == "devices": content = self.get_device_info() elif rcheck == "updater" and rtype in ['container', 'object']: content = self.get_updater_info(rtype) elif rcheck == "auditor" and rtype in all_rtypes: content = self.get_auditor_info(rtype) elif rcheck == "expirer" and rtype == 'object': content = self.get_expirer_info(rtype) elif rcheck == "mounted": content = self.get_mounted() elif rcheck == "unmounted": content = self.get_unmounted() elif rcheck == "diskusage": content = self.get_diskusage() elif rcheck == "ringmd5": content = self.get_ring_md5() elif rcheck == "quarantined": content = self.get_quarantine_count() elif rcheck == "sockstat": content = self.get_socket_info() elif rcheck == "version": content = self.get_version() else: content = "Invalid path: %s" % req.path return Response(request=req, status="404 Not Found", body=content, content_type="text/plain") if content is not None: return Response(request=req, body=json.dumps(content), content_type="application/json") else: return Response(request=req, status="500 Server Error", body="Internal server error.", content_type="text/plain") def __call__(self, env, start_response): req = Request(env) if req.path.startswith('/recon/'): return self.GET(req)(env, start_response) else: return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def recon_filter(app): return ReconMiddleware(app, conf) return recon_filter swift-1.13.1/swift/common/middleware/staticweb.py0000664000175400017540000005272212323703611023175 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ This StaticWeb WSGI middleware will serve container data as a static web site with index file and error file resolution and optional file listings. This mode is normally only active for anonymous requests. When using keystone for authentication set ``delay_auth_decision = true`` in the authtoken middleware configuration in your ``/etc/swift/proxy-server.conf`` file. If you want to use it with authenticated requests, set the ``X-Web-Mode: true`` header on the request. The ``staticweb`` filter should be added to the pipeline in your ``/etc/swift/proxy-server.conf`` file just after any auth middleware. Also, the configuration section for the ``staticweb`` middleware itself needs to be added. For example:: [DEFAULT] ... [pipeline:main] pipeline = catch_errors healthcheck proxy-logging cache ratelimit tempauth staticweb proxy-logging proxy-server ... [filter:staticweb] use = egg:swift#staticweb Any publicly readable containers (for example, ``X-Container-Read: .r:*``, see :ref:`acls` for more information on this) will be checked for X-Container-Meta-Web-Index and X-Container-Meta-Web-Error header values:: X-Container-Meta-Web-Index X-Container-Meta-Web-Error If X-Container-Meta-Web-Index is set, any files will be served without having to specify the part. For instance, setting ``X-Container-Meta-Web-Index: index.html`` will be able to serve the object .../pseudo/path/index.html with just .../pseudo/path or .../pseudo/path/ If X-Container-Meta-Web-Error is set, any errors (currently just 401 Unauthorized and 404 Not Found) will instead serve the .../ object. For instance, setting ``X-Container-Meta-Web-Error: error.html`` will serve .../404error.html for requests for paths not found. For pseudo paths that have no , this middleware can serve HTML file listings if you set the ``X-Container-Meta-Web-Listings: true`` metadata item on the container. If listings are enabled, the listings can have a custom style sheet by setting the X-Container-Meta-Web-Listings-CSS header. For instance, setting ``X-Container-Meta-Web-Listings-CSS: listing.css`` will make listings link to the .../listing.css style sheet. If you "view source" in your browser on a listing page, you will see the well defined document structure that can be styled. The content-type of directory marker objects can be modified by setting the ``X-Container-Meta-Web-Directory-Type`` header. If the header is not set, application/directory is used by default. Directory marker objects are 0-byte objects that represent directories to create a simulated hierarchical structure. Example usage of this middleware via ``swift``: Make the container publicly readable:: swift post -r '.r:*' container You should be able to get objects directly, but no index.html resolution or listings. Set an index file directive:: swift post -m 'web-index:index.html' container You should be able to hit paths that have an index.html without needing to type the index.html part. Turn on listings:: swift post -m 'web-listings: true' container Now you should see object listings for paths and pseudo paths that have no index.html. Enable a custom listings style sheet:: swift post -m 'web-listings-css:listings.css' container Set an error file:: swift post -m 'web-error:error.html' container Now 401's should load 401error.html, 404's should load 404error.html, etc. Set Content-Type of directory marker object:: swift post -m 'web-directory-type:text/directory' container Now 0-byte objects with a content-type of text/directory will be treated as directories rather than objects. """ import cgi import time from swift.common.utils import human_readable, split_path, config_true_value, \ json, quote, get_valid_utf8_str, register_swift_info from swift.common.wsgi import make_pre_authed_env, WSGIContext from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound from swift.proxy.controllers.base import get_container_info class _StaticWebContext(WSGIContext): """ The Static Web WSGI middleware filter; serves container data as a static web site. See `staticweb`_ for an overview. This _StaticWebContext is used by StaticWeb with each request that might need to be handled to make keeping contextual information about the request a bit simpler than storing it in the WSGI env. """ def __init__(self, staticweb, version, account, container, obj): WSGIContext.__init__(self, staticweb.app) self.version = version self.account = account self.container = container self.obj = obj self.app = staticweb.app self.agent = '%(orig)s StaticWeb' # Results from the last call to self._get_container_info. self._index = self._error = self._listings = self._listings_css = \ self._dir_type = None def _error_response(self, response, env, start_response): """ Sends the error response to the remote client, possibly resolving a custom error response body based on x-container-meta-web-error. :param response: The error response we should default to sending. :param env: The original request WSGI environment. :param start_response: The WSGI start_response hook. """ if not self._error: start_response(self._response_status, self._response_headers, self._response_exc_info) return response save_response_status = self._response_status save_response_headers = self._response_headers save_response_exc_info = self._response_exc_info resp = self._app_call(make_pre_authed_env( env, 'GET', '/%s/%s/%s/%s%s' % ( self.version, self.account, self.container, self._get_status_int(), self._error), self.agent, swift_source='SW')) if is_success(self._get_status_int()): start_response(save_response_status, self._response_headers, self._response_exc_info) return resp start_response(save_response_status, save_response_headers, save_response_exc_info) return response def _get_container_info(self, env): """ Retrieves x-container-meta-web-index, x-container-meta-web-error, x-container-meta-web-listings, x-container-meta-web-listings-css, and x-container-meta-web-directory-type from memcache or from the cluster and stores the result in memcache and in self._index, self._error, self._listings, self._listings_css and self._dir_type. :param env: The WSGI environment dict. """ self._index = self._error = self._listings = self._listings_css = \ self._dir_type = None container_info = get_container_info(env, self.app, swift_source='SW') if is_success(container_info['status']): meta = container_info.get('meta', {}) self._index = meta.get('web-index', '').strip() self._error = meta.get('web-error', '').strip() self._listings = meta.get('web-listings', '').strip() self._listings_css = meta.get('web-listings-css', '').strip() self._dir_type = meta.get('web-directory-type', '').strip() def _listing(self, env, start_response, prefix=None): """ Sends an HTML object listing to the remote client. :param env: The original WSGI environment dict. :param start_response: The original WSGI start_response hook. :param prefix: Any prefix desired for the container listing. """ if not config_true_value(self._listings): body = '\n' \ '\n' \ '\n' \ 'Listing of %s\n' % cgi.escape(env['PATH_INFO']) if self._listings_css: body += ' \n' % self._build_css_path(prefix or '') else: body += ' \n' body += '\n' \ '

Web Listing Disabled

' \ '

The owner of this web site has disabled web listing.' \ '

If you are the owner of this web site, you can enable' \ ' web listing by setting X-Container-Meta-Web-Listings.

' if self._index: body += '

Index File Not Found

' \ '

The owner of this web site has set ' \ ' X-Container-Meta-Web-Index: %s. ' \ ' However, this file is not found.

' % self._index body += ' \n\n' resp = HTTPNotFound(body=body)(env, self._start_response) return self._error_response(resp, env, start_response) tmp_env = make_pre_authed_env( env, 'GET', '/%s/%s/%s' % ( self.version, self.account, self.container), self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'delimiter=/&format=json' if prefix: tmp_env['QUERY_STRING'] += '&prefix=%s' % quote(prefix) else: prefix = '' resp = self._app_call(tmp_env) if not is_success(self._get_status_int()): return self._error_response(resp, env, start_response) listing = None body = ''.join(resp) if body: listing = json.loads(body) if not listing: resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) headers = {'Content-Type': 'text/html; charset=UTF-8'} body = '\n' \ '\n' \ ' \n' \ ' Listing of %s\n' % \ cgi.escape(env['PATH_INFO']) if self._listings_css: body += ' \n' % (self._build_css_path(prefix)) else: body += ' \n' body += ' \n' \ ' \n' \ '

Listing of %s

\n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' % \ cgi.escape(env['PATH_INFO']) if prefix: body += ' \n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' for item in listing: if 'subdir' in item: subdir = get_valid_utf8_str(item['subdir']) if prefix: subdir = subdir[len(prefix):] body += ' \n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' % \ (quote(subdir), cgi.escape(subdir)) for item in listing: if 'name' in item: name = get_valid_utf8_str(item['name']) if prefix: name = name[len(prefix):] content_type = get_valid_utf8_str(item['content_type']) bytes = get_valid_utf8_str(human_readable(item['bytes'])) last_modified = (cgi.escape(item['last_modified']). split('.')[0].replace('T', ' ')) body += ' \n' \ ' \n' \ ' \n' \ ' \n' \ ' \n' % \ (' '.join('type-' + cgi.escape(t.lower(), quote=True) for t in content_type.split('/')), quote(name), cgi.escape(name), bytes, get_valid_utf8_str(last_modified)) body += '
NameSizeDate
../  
%s  
%s%s%s
\n' \ ' \n' \ '\n' resp = Response(headers=headers, body=body) return resp(env, start_response) def _build_css_path(self, prefix=''): """ Constructs a relative path from a given prefix within the container. URLs and paths starting with '/' are not modified. :param prefix: The prefix for the container listing. """ if self._listings_css.startswith(('/', 'http://', 'https://')): css_path = quote(self._listings_css, ':/') else: css_path = '../' * prefix.count('/') + quote(self._listings_css) return css_path def handle_container(self, env, start_response): """ Handles a possible static web request for a container. :param env: The original WSGI environment dict. :param start_response: The original WSGI start_response hook. """ self._get_container_info(env) if not self._listings and not self._index: if config_true_value(env.get('HTTP_X_WEB_MODE', 'f')): return HTTPNotFound()(env, start_response) return self.app(env, start_response) if env['PATH_INFO'][-1] != '/': resp = HTTPMovedPermanently( location=(env['PATH_INFO'] + '/')) return resp(env, start_response) if not self._index: return self._listing(env, start_response) tmp_env = dict(env) tmp_env['HTTP_USER_AGENT'] = \ '%s StaticWeb' % env.get('HTTP_USER_AGENT') tmp_env['swift.source'] = 'SW' tmp_env['PATH_INFO'] += self._index resp = self._app_call(tmp_env) status_int = self._get_status_int() if status_int == HTTP_NOT_FOUND: return self._listing(env, start_response) elif not is_success(self._get_status_int()) and \ not is_redirection(self._get_status_int()): return self._error_response(resp, env, start_response) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp def handle_object(self, env, start_response): """ Handles a possible static web request for an object. This object could resolve into an index or listing request. :param env: The original WSGI environment dict. :param start_response: The original WSGI start_response hook. """ tmp_env = dict(env) tmp_env['HTTP_USER_AGENT'] = \ '%s StaticWeb' % env.get('HTTP_USER_AGENT') tmp_env['swift.source'] = 'SW' resp = self._app_call(tmp_env) status_int = self._get_status_int() self._get_container_info(env) if is_success(status_int) or is_redirection(status_int): # Treat directory marker objects as not found if not self._dir_type: self._dir_type = 'application/directory' content_length = self._response_header_value('content-length') content_length = int(content_length) if content_length else 0 if self._response_header_value('content-type') == self._dir_type \ and content_length <= 1: status_int = HTTP_NOT_FOUND else: start_response(self._response_status, self._response_headers, self._response_exc_info) return resp if status_int != HTTP_NOT_FOUND: # Retaining the previous code's behavior of not using custom error # pages for non-404 errors. self._error = None return self._error_response(resp, env, start_response) if not self._listings and not self._index: start_response(self._response_status, self._response_headers, self._response_exc_info) return resp status_int = HTTP_NOT_FOUND if self._index: tmp_env = dict(env) tmp_env['HTTP_USER_AGENT'] = \ '%s StaticWeb' % env.get('HTTP_USER_AGENT') tmp_env['swift.source'] = 'SW' if tmp_env['PATH_INFO'][-1] != '/': tmp_env['PATH_INFO'] += '/' tmp_env['PATH_INFO'] += self._index resp = self._app_call(tmp_env) status_int = self._get_status_int() if is_success(status_int) or is_redirection(status_int): if env['PATH_INFO'][-1] != '/': resp = HTTPMovedPermanently( location=env['PATH_INFO'] + '/') return resp(env, start_response) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp if status_int == HTTP_NOT_FOUND: if env['PATH_INFO'][-1] != '/': tmp_env = make_pre_authed_env( env, 'GET', '/%s/%s/%s' % ( self.version, self.account, self.container), self.agent, swift_source='SW') tmp_env['QUERY_STRING'] = 'limit=1&format=json&delimiter' \ '=/&limit=1&prefix=%s' % quote(self.obj + '/') resp = self._app_call(tmp_env) body = ''.join(resp) if not is_success(self._get_status_int()) or not body or \ not json.loads(body): resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) resp = HTTPMovedPermanently(location=env['PATH_INFO'] + '/') return resp(env, start_response) return self._listing(env, start_response, self.obj) class StaticWeb(object): """ The Static Web WSGI middleware filter; serves container data as a static web site. See `staticweb`_ for an overview. The proxy logs created for any subrequests made will have swift.source set to "SW". :param app: The next WSGI application/filter in the paste.deploy pipeline. :param conf: The filter configuration dict. """ def __init__(self, app, conf): #: The next WSGI application/filter in the paste.deploy pipeline. self.app = app #: The filter configuration dict. self.conf = conf def __call__(self, env, start_response): """ Main hook into the WSGI paste.deploy filter/app pipeline. :param env: The WSGI environment dict. :param start_response: The WSGI start_response hook. """ env['staticweb.start_time'] = time.time() try: (version, account, container, obj) = \ split_path(env['PATH_INFO'], 2, 4, True) except ValueError: return self.app(env, start_response) if env['REQUEST_METHOD'] not in ('HEAD', 'GET'): return self.app(env, start_response) if env.get('REMOTE_USER') and \ not config_true_value(env.get('HTTP_X_WEB_MODE', 'f')): return self.app(env, start_response) if not container: return self.app(env, start_response) context = _StaticWebContext(self, version, account, container, obj) if obj: return context.handle_object(env, start_response) return context.handle_container(env, start_response) def filter_factory(global_conf, **local_conf): """Returns a Static Web WSGI filter for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) register_swift_info('staticweb') def staticweb_filter(app): return StaticWeb(app, conf) return staticweb_filter swift-1.13.1/swift/common/middleware/domain_remap.py0000664000175400017540000001304612323703611023637 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Domain Remap Middleware Middleware that translates container and account parts of a domain to path parameters that the proxy server understands. container.account.storageurl/object gets translated to container.account.storageurl/path_root/account/container/object account.storageurl/path_root/container/object gets translated to account.storageurl/path_root/account/container/object Browsers can convert a host header to lowercase, so check that reseller prefix on the account is the correct case. This is done by comparing the items in the reseller_prefixes config option to the found prefix. If they match except for case, the item from reseller_prefixes will be used instead of the found reseller prefix. The reseller_prefixes list is exclusive. If defined, any request with an account prefix not in that list will be ignored by this middleware. reseller_prefixes defaults to 'AUTH'. Note that this middleware requires that container names and account names (except as described above) must be DNS-compatible. This means that the account name created in the system and the containers created by users cannot exceed 63 characters or have UTF-8 characters. These are restrictions over and above what swift requires and are not explicitly checked. Simply put, the this middleware will do a best-effort attempt to derive account and container names from elements in the domain name and put those derived values into the URL path (leaving the Host header unchanged). Also note that using container sync with remapped domain names is not advised. With container sync, you should use the true storage end points as sync destinations. """ from swift.common.swob import Request, HTTPBadRequest class DomainRemapMiddleware(object): """ Domain Remap Middleware See above for a full description. :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf): self.app = app self.storage_domain = conf.get('storage_domain', 'example.com') if self.storage_domain and self.storage_domain[0] != '.': self.storage_domain = '.' + self.storage_domain self.path_root = conf.get('path_root', 'v1').strip('/') prefixes = conf.get('reseller_prefixes', 'AUTH') self.reseller_prefixes = [x.strip() for x in prefixes.split(',') if x.strip()] self.reseller_prefixes_lower = [x.lower() for x in self.reseller_prefixes] def __call__(self, env, start_response): if not self.storage_domain: return self.app(env, start_response) if 'HTTP_HOST' in env: given_domain = env['HTTP_HOST'] else: given_domain = env['SERVER_NAME'] port = '' if ':' in given_domain: given_domain, port = given_domain.rsplit(':', 1) if given_domain.endswith(self.storage_domain): parts_to_parse = given_domain[:-len(self.storage_domain)] parts_to_parse = parts_to_parse.strip('.').split('.') len_parts_to_parse = len(parts_to_parse) if len_parts_to_parse == 2: container, account = parts_to_parse elif len_parts_to_parse == 1: container, account = None, parts_to_parse[0] else: resp = HTTPBadRequest(request=Request(env), body='Bad domain in host header', content_type='text/plain') return resp(env, start_response) if '_' not in account and '-' in account: account = account.replace('-', '_', 1) account_reseller_prefix = account.split('_', 1)[0].lower() if account_reseller_prefix not in self.reseller_prefixes_lower: # account prefix is not in config list. bail. return self.app(env, start_response) prefix_index = self.reseller_prefixes_lower.index( account_reseller_prefix) real_prefix = self.reseller_prefixes[prefix_index] if not account.startswith(real_prefix): account_suffix = account[len(real_prefix):] account = real_prefix + account_suffix path = env['PATH_INFO'].strip('/') new_path_parts = ['', self.path_root, account] if container: new_path_parts.append(container) if path.startswith(self.path_root): path = path[len(self.path_root):].lstrip('/') if path: new_path_parts.append(path) new_path = '/'.join(new_path_parts) env['PATH_INFO'] = new_path return self.app(env, start_response) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def domain_filter(app): return DomainRemapMiddleware(app, conf) return domain_filter swift-1.13.1/swift/common/middleware/__init__.py0000664000175400017540000000000012323703611022725 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/common/middleware/cname_lookup.py0000664000175400017540000001531412323703611023660 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ CNAME Lookup Middleware Middleware that translates an unknown domain in the host header to something that ends with the configured storage_domain by looking up the given domain's CNAME record in DNS. This middleware will continue to follow a CNAME chain in DNS until it finds a record ending in the configured storage domain or it reaches the configured maximum lookup depth. If a match is found, the environment's Host header is rewritten and the request is passed further down the WSGI chain. """ import socket from swift import gettext_ as _ try: import dns.resolver from dns.exception import DNSException from dns.resolver import NXDOMAIN, NoAnswer except ImportError: # catch this to allow docs to be built without the dependency MODULE_DEPENDENCY_MET = False else: # executed if the try block finishes with no errors MODULE_DEPENDENCY_MET = True from swift.common.swob import Request, HTTPBadRequest from swift.common.utils import cache_from_env, get_logger, list_from_csv def lookup_cname(domain): # pragma: no cover """ Given a domain, returns its DNS CNAME mapping and DNS ttl. :param domain: domain to query on :returns: (ttl, result) """ try: answer = dns.resolver.query(domain, 'CNAME').rrset ttl = answer.ttl result = answer.items[0].to_text() result = result.rstrip('.') return ttl, result except (DNSException, NXDOMAIN, NoAnswer): return 0, None def is_ip(domain): try: socket.inet_pton(socket.AF_INET, domain) return True except socket.error: try: socket.inet_pton(socket.AF_INET6, domain) return True except socket.error: return False class CNAMELookupMiddleware(object): """ CNAME Lookup Middleware See above for a full description. :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf): if not MODULE_DEPENDENCY_MET: # reraise the exception if the dependency wasn't met raise ImportError('dnspython is required for this module') self.app = app storage_domain = conf.get('storage_domain', 'example.com') self.storage_domain = ['.' + s for s in list_from_csv(storage_domain) if not s.startswith('.')] self.storage_domain += [s for s in list_from_csv(storage_domain) if s.startswith('.')] self.lookup_depth = int(conf.get('lookup_depth', '1')) self.memcache = None self.logger = get_logger(conf, log_route='cname-lookup') def _domain_endswith_in_storage_domain(self, a_domain): for domain in self.storage_domain: if a_domain.endswith(domain): return True return False def __call__(self, env, start_response): if not self.storage_domain: return self.app(env, start_response) if 'HTTP_HOST' in env: given_domain = env['HTTP_HOST'] else: given_domain = env['SERVER_NAME'] port = '' if ':' in given_domain: given_domain, port = given_domain.rsplit(':', 1) if given_domain == self.storage_domain[1:]: # strip initial '.' return self.app(env, start_response) if is_ip(given_domain): return self.app(env, start_response) a_domain = given_domain if not self._domain_endswith_in_storage_domain(a_domain): if self.memcache is None: self.memcache = cache_from_env(env) error = True for tries in xrange(self.lookup_depth): found_domain = None if self.memcache: memcache_key = ''.join(['cname-', a_domain]) found_domain = self.memcache.get(memcache_key) if not found_domain: ttl, found_domain = lookup_cname(a_domain) if self.memcache: memcache_key = ''.join(['cname-', given_domain]) self.memcache.set(memcache_key, found_domain, time=ttl) if found_domain is None or found_domain == a_domain: # no CNAME records or we're at the last lookup error = True found_domain = None break elif self._domain_endswith_in_storage_domain(found_domain): # Found it! self.logger.info( _('Mapped %(given_domain)s to %(found_domain)s') % {'given_domain': given_domain, 'found_domain': found_domain}) if port: env['HTTP_HOST'] = ':'.join([found_domain, port]) else: env['HTTP_HOST'] = found_domain error = False break else: # try one more deep in the chain self.logger.debug( _('Following CNAME chain for ' '%(given_domain)s to %(found_domain)s') % {'given_domain': given_domain, 'found_domain': found_domain}) a_domain = found_domain if error: if found_domain: msg = 'CNAME lookup failed after %d tries' % \ self.lookup_depth else: msg = 'CNAME lookup failed to resolve to a valid domain' resp = HTTPBadRequest(request=Request(env), body=msg, content_type='text/plain') return resp(env, start_response) return self.app(env, start_response) def filter_factory(global_conf, **local_conf): # pragma: no cover conf = global_conf.copy() conf.update(local_conf) def cname_filter(app): return CNAMELookupMiddleware(app, conf) return cname_filter swift-1.13.1/swift/common/constraints.py0000664000175400017540000002241112323703614021435 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import urllib from urllib import unquote from ConfigParser import ConfigParser, NoSectionError, NoOptionError from swift.common.utils import ismount, split_path, SWIFT_CONF_FILE from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \ HTTPRequestEntityTooLarge, HTTPPreconditionFailed MAX_FILE_SIZE = 5368709122 MAX_META_NAME_LENGTH = 128 MAX_META_VALUE_LENGTH = 256 MAX_META_COUNT = 90 MAX_META_OVERALL_SIZE = 4096 MAX_HEADER_SIZE = 8192 MAX_OBJECT_NAME_LENGTH = 1024 CONTAINER_LISTING_LIMIT = 10000 ACCOUNT_LISTING_LIMIT = 10000 MAX_ACCOUNT_NAME_LENGTH = 256 MAX_CONTAINER_NAME_LENGTH = 256 DEFAULT_CONSTRAINTS = { 'max_file_size': MAX_FILE_SIZE, 'max_meta_name_length': MAX_META_NAME_LENGTH, 'max_meta_value_length': MAX_META_VALUE_LENGTH, 'max_meta_count': MAX_META_COUNT, 'max_meta_overall_size': MAX_META_OVERALL_SIZE, 'max_header_size': MAX_HEADER_SIZE, 'max_object_name_length': MAX_OBJECT_NAME_LENGTH, 'container_listing_limit': CONTAINER_LISTING_LIMIT, 'account_listing_limit': ACCOUNT_LISTING_LIMIT, 'max_account_name_length': MAX_ACCOUNT_NAME_LENGTH, 'max_container_name_length': MAX_CONTAINER_NAME_LENGTH, } SWIFT_CONSTRAINTS_LOADED = False OVERRIDE_CONSTRAINTS = {} # any constraints overridden by SWIFT_CONF_FILE EFFECTIVE_CONSTRAINTS = {} # populated by reload_constraints def reload_constraints(): """ Parse SWIFT_CONF_FILE and reset module level global contraint attrs, populating OVERRIDE_CONSTRAINTS AND EFFECTIVE_CONSTRAINTS along the way. """ global SWIFT_CONSTRAINTS_LOADED, OVERRIDE_CONSTRAINTS SWIFT_CONSTRAINTS_LOADED = False OVERRIDE_CONSTRAINTS = {} constraints_conf = ConfigParser() if constraints_conf.read(SWIFT_CONF_FILE): SWIFT_CONSTRAINTS_LOADED = True for name in DEFAULT_CONSTRAINTS: try: value = int(constraints_conf.get('swift-constraints', name)) except (NoSectionError, NoOptionError): pass else: OVERRIDE_CONSTRAINTS[name] = value for name, default in DEFAULT_CONSTRAINTS.items(): value = OVERRIDE_CONSTRAINTS.get(name, default) EFFECTIVE_CONSTRAINTS[name] = value # "globals" in this context is module level globals, always. globals()[name.upper()] = value reload_constraints() # Maximum slo segments in buffer MAX_BUFFERED_SLO_SEGMENTS = 10000 #: Query string format= values to their corresponding content-type values FORMAT2CONTENT_TYPE = {'plain': 'text/plain', 'json': 'application/json', 'xml': 'application/xml'} def check_metadata(req, target_type): """ Check metadata sent in the request headers. :param req: request object :param target_type: str: one of: object, container, or account: indicates which type the target storage for the metadata is :returns: HTTPBadRequest with bad metadata otherwise None """ prefix = 'x-%s-meta-' % target_type.lower() meta_count = 0 meta_size = 0 for key, value in req.headers.iteritems(): if isinstance(value, basestring) and len(value) > MAX_HEADER_SIZE: return HTTPBadRequest(body='Header value too long: %s' % key[:MAX_META_NAME_LENGTH], request=req, content_type='text/plain') if not key.lower().startswith(prefix): continue key = key[len(prefix):] if not key: return HTTPBadRequest(body='Metadata name cannot be empty', request=req, content_type='text/plain') meta_count += 1 meta_size += len(key) + len(value) if len(key) > MAX_META_NAME_LENGTH: return HTTPBadRequest( body='Metadata name too long: %s%s' % (prefix, key), request=req, content_type='text/plain') elif len(value) > MAX_META_VALUE_LENGTH: return HTTPBadRequest( body='Metadata value longer than %d: %s%s' % ( MAX_META_VALUE_LENGTH, prefix, key), request=req, content_type='text/plain') elif meta_count > MAX_META_COUNT: return HTTPBadRequest( body='Too many metadata items; max %d' % MAX_META_COUNT, request=req, content_type='text/plain') elif meta_size > MAX_META_OVERALL_SIZE: return HTTPBadRequest( body='Total metadata too large; max %d' % MAX_META_OVERALL_SIZE, request=req, content_type='text/plain') return None def check_object_creation(req, object_name): """ Check to ensure that everything is alright about an object to be created. :param req: HTTP request object :param object_name: name of object to be created :returns HTTPRequestEntityTooLarge: the object is too large :returns HTTPLengthRequired: missing content-length header and not a chunked request :returns HTTPBadRequest: missing or bad content-type header, or bad metadata """ if req.content_length and req.content_length > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(body='Your request is too large.', request=req, content_type='text/plain') if req.content_length is None and \ req.headers.get('transfer-encoding') != 'chunked': return HTTPLengthRequired(request=req) if 'X-Copy-From' in req.headers and req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte body', request=req, content_type='text/plain') if len(object_name) > MAX_OBJECT_NAME_LENGTH: return HTTPBadRequest(body='Object name length of %d longer than %d' % (len(object_name), MAX_OBJECT_NAME_LENGTH), request=req, content_type='text/plain') if 'Content-Type' not in req.headers: return HTTPBadRequest(request=req, content_type='text/plain', body='No content type') if not check_utf8(req.headers['Content-Type']): return HTTPBadRequest(request=req, body='Invalid Content-Type', content_type='text/plain') return check_metadata(req, 'object') def check_mount(root, drive): """ Verify that the path to the device is a mount point and mounted. This allows us to fast fail on drives that have been unmounted because of issues, and also prevents us for accidentally filling up the root partition. :param root: base path where the devices are mounted :param drive: drive name to be checked :returns: True if it is a valid mounted device, False otherwise """ if not (urllib.quote_plus(drive) == drive): return False path = os.path.join(root, drive) return ismount(path) def check_float(string): """ Helper function for checking if a string can be converted to a float. :param string: string to be verified as a float :returns: True if the string can be converted to a float, False otherwise """ try: float(string) return True except ValueError: return False def check_utf8(string): """ Validate if a string is valid UTF-8 str or unicode and that it does not contain any null character. :param string: string to be validated :returns: True if the string is valid utf-8 str or unicode and contains no null characters, False otherwise """ if not string: return False try: if isinstance(string, unicode): string.encode('utf-8') else: string.decode('UTF-8') return '\x00' not in string # If string is unicode, decode() will raise UnicodeEncodeError # So, we should catch both UnicodeDecodeError & UnicodeEncodeError except UnicodeError: return False def check_copy_from_header(req): """ Validate that the value from x-copy-from header is well formatted. We assume the caller ensures that x-copy-from header is present in req.headers. :param req: HTTP request object :returns: A tuple with container name and object name :raise: HTTPPreconditionFailed if x-copy-from value is not well formatted. """ src_header = unquote(req.headers.get('X-Copy-From')) if not src_header.startswith('/'): src_header = '/' + src_header try: return split_path(src_header, 2, 2, True) except ValueError: raise HTTPPreconditionFailed( request=req, body='X-Copy-From header must be of the form' '/') swift-1.13.1/swift/common/swob.py0000664000175400017540000013771712323703614020060 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Implementation of WSGI Request and Response objects. This library has a very similar API to Webob. It wraps WSGI request environments and response values into objects that are more friendly to interact with. Why Swob and not just use WebOb? By Michael Barton We used webob for years. The main problem was that the interface wasn't stable. For a while, each of our several test suites required a slightly different version of webob to run, and none of them worked with the then-current version. It was a huge headache, so we just scrapped it. This is kind of a ton of code, but it's also been a huge relief to not have to scramble to add a bunch of code branches all over the place to keep Swift working every time webob decides some interface needs to change. """ from collections import defaultdict from cStringIO import StringIO import UserDict import time from functools import partial from datetime import datetime, timedelta, tzinfo from email.utils import parsedate import urlparse import urllib2 import re import random import functools import inspect from swift.common.utils import reiterate, split_path RESPONSE_REASONS = { 100: ('Continue', ''), 200: ('OK', ''), 201: ('Created', ''), 202: ('Accepted', 'The request is accepted for processing.'), 204: ('No Content', ''), 206: ('Partial Content', ''), 301: ('Moved Permanently', 'The resource has moved permanently.'), 302: ('Found', 'The resource has moved temporarily.'), 303: ('See Other', 'The response to the request can be found under a ' 'different URI.'), 304: ('Not Modified', ''), 307: ('Temporary Redirect', 'The resource has moved temporarily.'), 400: ('Bad Request', 'The server could not comply with the request since ' 'it is either malformed or otherwise incorrect.'), 401: ('Unauthorized', 'This server could not verify that you are ' 'authorized to access the document you requested.'), 402: ('Payment Required', 'Access was denied for financial reasons.'), 403: ('Forbidden', 'Access was denied to this resource.'), 404: ('Not Found', 'The resource could not be found.'), 405: ('Method Not Allowed', 'The method is not allowed for this ' 'resource.'), 406: ('Not Acceptable', 'The resource is not available in a format ' 'acceptable to your browser.'), 408: ('Request Timeout', 'The server has waited too long for the request ' 'to be sent by the client.'), 409: ('Conflict', 'There was a conflict when trying to complete ' 'your request.'), 410: ('Gone', 'This resource is no longer available.'), 411: ('Length Required', 'Content-Length header required.'), 412: ('Precondition Failed', 'A precondition for this request was not ' 'met.'), 413: ('Request Entity Too Large', 'The body of your request was too ' 'large for this server.'), 414: ('Request URI Too Long', 'The request URI was too long for this ' 'server.'), 415: ('Unsupported Media Type', 'The request media type is not ' 'supported by this server.'), 416: ('Requested Range Not Satisfiable', 'The Range requested is not ' 'available.'), 417: ('Expectation Failed', 'Expectation failed.'), 422: ('Unprocessable Entity', 'Unable to process the contained ' 'instructions'), 499: ('Client Disconnect', 'The client was disconnected during request.'), 500: ('Internal Error', 'The server has either erred or is incapable of ' 'performing the requested operation.'), 501: ('Not Implemented', 'The requested method is not implemented by ' 'this server.'), 502: ('Bad Gateway', 'Bad gateway.'), 503: ('Service Unavailable', 'The server is currently unavailable. ' 'Please try again at a later time.'), 504: ('Gateway Timeout', 'A timeout has occurred speaking to a ' 'backend server.'), 507: ('Insufficient Storage', 'There was not enough space to save the ' 'resource. Drive: %(drive)s'), } class _UTC(tzinfo): """ A tzinfo class for datetime objects that returns a 0 timedelta (UTC time) """ def dst(self, dt): return timedelta(0) utcoffset = dst def tzname(self, dt): return 'UTC' UTC = _UTC() def _datetime_property(header): """ Set and retrieve the datetime value of self.headers[header] (Used by both request and response) The header is parsed on retrieval and a datetime object is returned. The header can be set using a datetime, numeric value, or str. If a value of None is given, the header is deleted. :param header: name of the header, e.g. "Content-Length" """ def getter(self): value = self.headers.get(header, None) if value is not None: try: parts = parsedate(self.headers[header])[:7] return datetime(*(parts + (UTC,))) except Exception: return None def setter(self, value): if isinstance(value, (float, int, long)): self.headers[header] = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(value)) elif isinstance(value, datetime): self.headers[header] = value.strftime("%a, %d %b %Y %H:%M:%S GMT") else: self.headers[header] = value return property(getter, setter, doc=("Retrieve and set the %s header as a datetime, " "set it with a datetime, int, or str") % header) def _header_property(header): """ Set and retrieve the value of self.headers[header] (Used by both request and response) If a value of None is given, the header is deleted. :param header: name of the header, e.g. "Transfer-Encoding" """ def getter(self): return self.headers.get(header, None) def setter(self, value): self.headers[header] = value return property(getter, setter, doc="Retrieve and set the %s header" % header) def _header_int_property(header): """ Set and retrieve the value of self.headers[header] (Used by both request and response) On retrieval, it converts values to integers. If a value of None is given, the header is deleted. :param header: name of the header, e.g. "Content-Length" """ def getter(self): val = self.headers.get(header, None) if val is not None: val = int(val) return val def setter(self, value): self.headers[header] = value return property(getter, setter, doc="Retrieve and set the %s header as an int" % header) class HeaderEnvironProxy(UserDict.DictMixin): """ A dict-like object that proxies requests to a wsgi environ, rewriting header keys to environ keys. For example, headers['Content-Range'] sets and gets the value of headers.environ['HTTP_CONTENT_RANGE'] """ def __init__(self, environ): self.environ = environ def _normalize(self, key): key = 'HTTP_' + key.replace('-', '_').upper() if key == 'HTTP_CONTENT_LENGTH': return 'CONTENT_LENGTH' if key == 'HTTP_CONTENT_TYPE': return 'CONTENT_TYPE' return key def __getitem__(self, key): return self.environ[self._normalize(key)] def __setitem__(self, key, value): if value is None: self.environ.pop(self._normalize(key), None) elif isinstance(value, unicode): self.environ[self._normalize(key)] = value.encode('utf-8') else: self.environ[self._normalize(key)] = str(value) def __contains__(self, key): return self._normalize(key) in self.environ def __delitem__(self, key): del self.environ[self._normalize(key)] def keys(self): keys = [key[5:].replace('_', '-').title() for key in self.environ if key.startswith('HTTP_')] if 'CONTENT_LENGTH' in self.environ: keys.append('Content-Length') if 'CONTENT_TYPE' in self.environ: keys.append('Content-Type') return keys class HeaderKeyDict(dict): """ A dict that title-cases all keys on the way in, so as to be case-insensitive. """ def __init__(self, base_headers=None, **kwargs): if base_headers: self.update(base_headers) self.update(kwargs) def update(self, other): if hasattr(other, 'keys'): for key in other.keys(): self[key.title()] = other[key] else: for key, value in other: self[key.title()] = value def __getitem__(self, key): return dict.get(self, key.title()) def __setitem__(self, key, value): if value is None: self.pop(key.title(), None) elif isinstance(value, unicode): return dict.__setitem__(self, key.title(), value.encode('utf-8')) else: return dict.__setitem__(self, key.title(), str(value)) def __contains__(self, key): return dict.__contains__(self, key.title()) def __delitem__(self, key): return dict.__delitem__(self, key.title()) def get(self, key, default=None): return dict.get(self, key.title(), default) def setdefault(self, key, value=None): if key not in self: self[key] = value return self[key] def pop(self, key, default=None): return dict.pop(self, key.title(), default) def _resp_status_property(): """ Set and retrieve the value of Response.status On retrieval, it concatenates status_int and title. When set to a str, it splits status_int and title apart. When set to an integer, retrieves the correct title for that response code from the RESPONSE_REASONS dict. """ def getter(self): return '%s %s' % (self.status_int, self.title) def setter(self, value): if isinstance(value, (int, long)): self.status_int = value self.explanation = self.title = RESPONSE_REASONS[value][0] else: if isinstance(value, unicode): value = value.encode('utf-8') self.status_int = int(value.split(' ', 1)[0]) self.explanation = self.title = value.split(' ', 1)[1] return property(getter, setter, doc="Retrieve and set the Response status, e.g. '200 OK'") def _resp_body_property(): """ Set and retrieve the value of Response.body If necessary, it will consume Response.app_iter to create a body. On assignment, encodes unicode values to utf-8, and sets the content-length to the length of the str. """ def getter(self): if not self._body: if not self._app_iter: return '' self._body = ''.join(self._app_iter) self._app_iter = None return self._body def setter(self, value): if isinstance(value, unicode): value = value.encode('utf-8') if isinstance(value, str): self.content_length = len(value) self._app_iter = None self._body = value return property(getter, setter, doc="Retrieve and set the Response body str") def _resp_etag_property(): """ Set and retrieve Response.etag This may be broken for etag use cases other than Swift's. Quotes strings when assigned and unquotes when read, for compatibility with webob. """ def getter(self): etag = self.headers.get('etag', None) if etag: etag = etag.replace('"', '') return etag def setter(self, value): if value is None: self.headers['etag'] = None else: self.headers['etag'] = '"%s"' % value return property(getter, setter, doc="Retrieve and set the response Etag header") def _resp_content_type_property(): """ Set and retrieve Response.content_type Strips off any charset when retrieved -- that is accessible via Response.charset. """ def getter(self): if 'content-type' in self.headers: return self.headers.get('content-type').split(';')[0] def setter(self, value): self.headers['content-type'] = value return property(getter, setter, doc="Retrieve and set the response Content-Type header") def _resp_charset_property(): """ Set and retrieve Response.charset On retrieval, separates the charset from the content-type. On assignment, removes any existing charset from the content-type and appends the new one. """ def getter(self): if '; charset=' in self.headers['content-type']: return self.headers['content-type'].split('; charset=')[1] def setter(self, value): if 'content-type' in self.headers: self.headers['content-type'] = self.headers['content-type'].split( ';')[0] if value: self.headers['content-type'] += '; charset=' + value return property(getter, setter, doc="Retrieve and set the response charset") def _resp_app_iter_property(): """ Set and retrieve Response.app_iter Mostly a pass-through to Response._app_iter; it's a property so it can zero out an existing content-length on assignment. """ def getter(self): return self._app_iter def setter(self, value): if isinstance(value, (list, tuple)): self.content_length = sum(map(len, value)) elif value is not None: self.content_length = None self._body = None self._app_iter = value return property(getter, setter, doc="Retrieve and set the response app_iter") def _req_fancy_property(cls, header, even_if_nonexistent=False): """ Set and retrieve "fancy" properties. On retrieval, these properties return a class that takes the value of the header as the only argument to their constructor. For assignment, those classes should implement a __str__ that converts them back to their header values. :param header: name of the header, e.g. "Accept" :param even_if_nonexistent: Return a value even if the header does not exist. Classes using this should be prepared to accept None as a parameter. """ def getter(self): try: if header in self.headers or even_if_nonexistent: return cls(self.headers.get(header)) except ValueError: return None def setter(self, value): self.headers[header] = value return property(getter, setter, doc=("Retrieve and set the %s " "property in the WSGI environ, as a %s object") % (header, cls.__name__)) class Range(object): """ Wraps a Request's Range header as a friendly object. After initialization, "range.ranges" is populated with a list of (start, end) tuples denoting the requested ranges. If there were any syntactically-invalid byte-range-spec values, "range.ranges" will be an empty list, per the relevant RFC: "The recipient of a byte-range-set that includes one or more syntactically invalid byte-range-spec values MUST ignore the header field that includes that byte-range-set." According to the RFC 2616 specification, the following cases will be all considered as syntactically invalid, thus, a ValueError is thrown so that the range header will be ignored. If the range value contains at least one of the following cases, the entire range is considered invalid, ValueError will be thrown so that the header will be ignored. 1. value not starts with bytes= 2. range value start is greater than the end, eg. bytes=5-3 3. range does not have start or end, eg. bytes=- 4. range does not have hyphen, eg. bytes=45 5. range value is non numeric 6. any combination of the above Every syntactically valid range will be added into the ranges list even when some of the ranges may not be satisfied by underlying content. :param headerval: value of the header as a str """ def __init__(self, headerval): headerval = headerval.replace(' ', '') if not headerval.lower().startswith('bytes='): raise ValueError('Invalid Range header: %s' % headerval) self.ranges = [] for rng in headerval[6:].split(','): # Check if the range has required hyphen. if rng.find('-') == -1: raise ValueError('Invalid Range header: %s' % headerval) start, end = rng.split('-', 1) if start: # when start contains non numeric value, this also causes # ValueError start = int(start) else: start = None if end: # when end contains non numeric value, this also causes # ValueError end = int(end) if start is not None and end < start: raise ValueError('Invalid Range header: %s' % headerval) else: end = None if start is None: raise ValueError('Invalid Range header: %s' % headerval) self.ranges.append((start, end)) def __str__(self): string = 'bytes=' for start, end in self.ranges: if start is not None: string += str(start) string += '-' if end is not None: string += str(end) string += ',' return string.rstrip(',') def ranges_for_length(self, length): """ This method is used to return multiple ranges for a given length which should represent the length of the underlying content. The constructor method __init__ made sure that any range in ranges list is syntactically valid. So if length is None or size of the ranges is zero, then the Range header should be ignored which will eventually make the response to be 200. If an empty list is returned by this method, it indicates that there are unsatisfiable ranges found in the Range header, 416 will be returned. if a returned list has at least one element, the list indicates that there is at least one range valid and the server should serve the request with a 206 status code. The start value of each range represents the starting position in the content, the end value represents the ending position. This method purposely adds 1 to the end number because the spec defines the Range to be inclusive. The Range spec can be found at the following link: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.1 :param length: length of the underlying content """ # not syntactically valid ranges, must ignore if length is None or not self.ranges or self.ranges == []: return None all_ranges = [] for single_range in self.ranges: begin, end = single_range # The possible values for begin and end are # None, 0, or a positive numeric number if begin is None: if end == 0: # this is the bytes=-0 case continue elif end > length: # This is the case where the end is greater than the # content length, as the RFC 2616 stated, the entire # content should be returned. all_ranges.append((0, length)) else: all_ranges.append((length - end, length)) continue # begin can only be 0 and numeric value from this point on if end is None: if begin < length: all_ranges.append((begin, length)) else: # the begin position is greater than or equal to the # content length; skip and move on to the next range continue # end can only be 0 or numeric value elif begin < length: # the begin position is valid, take the min of end + 1 or # the total length of the content all_ranges.append((begin, min(end + 1, length))) return all_ranges class Match(object): """ Wraps a Request's If-[None-]Match header as a friendly object. :param headerval: value of the header as a str """ def __init__(self, headerval): self.tags = set() for tag in headerval.split(', '): if tag.startswith('"') and tag.endswith('"'): self.tags.add(tag[1:-1]) else: self.tags.add(tag) def __contains__(self, val): return '*' in self.tags or val in self.tags class Accept(object): """ Wraps a Request's Accept header as a friendly object. :param headerval: value of the header as a str """ # RFC 2616 section 2.2 token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+' qdtext = r'[^"]' quoted_pair = r'(?:\\.)' quoted_string = r'"(?:' + qdtext + r'|' + quoted_pair + r')*"' extension = (r'(?:\s*;\s*(?:' + token + r")\s*=\s*" + r'(?:' + token + r'|' + quoted_string + r'))') acc = (r'^\s*(' + token + r')/(' + token + r')(' + extension + r'*?\s*)$') acc_pattern = re.compile(acc) def __init__(self, headerval): self.headerval = headerval def _get_types(self): types = [] if not self.headerval: return [] for typ in self.headerval.split(','): type_parms = self.acc_pattern.findall(typ) if not type_parms: raise ValueError('Invalid accept header') typ, subtype, parms = type_parms[0] parms = [p.strip() for p in parms.split(';') if p.strip()] seen_q_already = False quality = 1.0 for parm in parms: name, value = parm.split('=') name = name.strip() value = value.strip() if name == 'q': if seen_q_already: raise ValueError('Multiple "q" params') seen_q_already = True quality = float(value) pattern = '^' + \ (self.token if typ == '*' else re.escape(typ)) + '/' + \ (self.token if subtype == '*' else re.escape(subtype)) + '$' types.append((pattern, quality, '*' not in (typ, subtype))) # sort candidates by quality, then whether or not there were globs types.sort(reverse=True, key=lambda t: (t[1], t[2])) return [t[0] for t in types] def best_match(self, options): """ Returns the item from "options" that best matches the accept header. Returns None if no available options are acceptable to the client. :param options: a list of content-types the server can respond with """ try: types = self._get_types() except ValueError: return None if not types and options: return options[0] for pattern in types: for option in options: if re.match(pattern, option): return option return None def __repr__(self): return self.headerval def _req_environ_property(environ_field): """ Set and retrieve value of the environ_field entry in self.environ. (Used by both request and response) """ def getter(self): return self.environ.get(environ_field, None) def setter(self, value): if isinstance(value, unicode): self.environ[environ_field] = value.encode('utf-8') else: self.environ[environ_field] = value return property(getter, setter, doc=("Get and set the %s property " "in the WSGI environment") % environ_field) def _req_body_property(): """ Set and retrieve the Request.body parameter. It consumes wsgi.input and returns the results. On assignment, uses a StringIO to create a new wsgi.input. """ def getter(self): body = self.environ['wsgi.input'].read() self.environ['wsgi.input'] = StringIO(body) return body def setter(self, value): self.environ['wsgi.input'] = StringIO(value) self.environ['CONTENT_LENGTH'] = str(len(value)) return property(getter, setter, doc="Get and set the request body str") def _host_url_property(): """ Retrieves the best guess that can be made for an absolute location up to the path, for example: https://host.com:1234 """ def getter(self): if 'HTTP_HOST' in self.environ: host = self.environ['HTTP_HOST'] else: host = '%s:%s' % (self.environ['SERVER_NAME'], self.environ['SERVER_PORT']) scheme = self.environ.get('wsgi.url_scheme', 'http') if scheme == 'http' and host.endswith(':80'): host, port = host.rsplit(':', 1) elif scheme == 'https' and host.endswith(':443'): host, port = host.rsplit(':', 1) return '%s://%s' % (scheme, host) return property(getter, doc="Get url for request/response up to path") class Request(object): """ WSGI Request object. """ range = _req_fancy_property(Range, 'range') if_none_match = _req_fancy_property(Match, 'if-none-match') accept = _req_fancy_property(Accept, 'accept', True) method = _req_environ_property('REQUEST_METHOD') referrer = referer = _req_environ_property('HTTP_REFERER') script_name = _req_environ_property('SCRIPT_NAME') path_info = _req_environ_property('PATH_INFO') host = _req_environ_property('HTTP_HOST') host_url = _host_url_property() remote_addr = _req_environ_property('REMOTE_ADDR') remote_user = _req_environ_property('REMOTE_USER') user_agent = _req_environ_property('HTTP_USER_AGENT') query_string = _req_environ_property('QUERY_STRING') if_match = _req_fancy_property(Match, 'if-match') body_file = _req_environ_property('wsgi.input') content_length = _header_int_property('content-length') if_modified_since = _datetime_property('if-modified-since') if_unmodified_since = _datetime_property('if-unmodified-since') body = _req_body_property() charset = None _params_cache = None acl = _req_environ_property('swob.ACL') def __init__(self, environ): self.environ = environ self.headers = HeaderEnvironProxy(self.environ) @classmethod def blank(cls, path, environ=None, headers=None, body=None, **kwargs): """ Create a new request object with the given parameters, and an environment otherwise filled in with non-surprising default values. :param path: encoded, parsed, and unquoted into PATH_INFO :param environ: WSGI environ dictionary :param headers: HTTP headers :param body: stuffed in a StringIO and hung on wsgi.input :param kwargs: any environ key with an property setter """ headers = headers or {} environ = environ or {} if isinstance(path, unicode): path = path.encode('utf-8') parsed_path = urlparse.urlparse(path) server_name = 'localhost' if parsed_path.netloc: server_name = parsed_path.netloc.split(':', 1)[0] server_port = parsed_path.port if server_port is None: server_port = {'http': 80, 'https': 443}.get(parsed_path.scheme, 80) if parsed_path.scheme and parsed_path.scheme not in ['http', 'https']: raise TypeError('Invalid scheme: %s' % parsed_path.scheme) env = { 'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'QUERY_STRING': parsed_path.query, 'PATH_INFO': urllib2.unquote(parsed_path.path), 'SERVER_NAME': server_name, 'SERVER_PORT': str(server_port), 'HTTP_HOST': '%s:%d' % (server_name, server_port), 'SERVER_PROTOCOL': 'HTTP/1.0', 'wsgi.version': (1, 0), 'wsgi.url_scheme': parsed_path.scheme or 'http', 'wsgi.errors': StringIO(''), 'wsgi.multithread': False, 'wsgi.multiprocess': False } env.update(environ) if body is not None: env['wsgi.input'] = StringIO(body) env['CONTENT_LENGTH'] = str(len(body)) elif 'wsgi.input' not in env: env['wsgi.input'] = StringIO('') req = Request(env) for key, val in headers.iteritems(): req.headers[key] = val for key, val in kwargs.items(): prop = getattr(Request, key, None) if prop and isinstance(prop, property): try: setattr(req, key, val) except AttributeError: pass else: continue raise TypeError("got unexpected keyword argument %r" % key) return req @property def params(self): "Provides QUERY_STRING parameters as a dictionary" if self._params_cache is None: if 'QUERY_STRING' in self.environ: self._params_cache = dict( urlparse.parse_qsl(self.environ['QUERY_STRING'], True)) else: self._params_cache = {} return self._params_cache str_params = params @property def path_qs(self): """The path of the request, without host but with query string.""" path = self.path if self.query_string: path += '?' + self.query_string return path @property def path(self): "Provides the full path of the request, excluding the QUERY_STRING" return urllib2.quote(self.environ.get('SCRIPT_NAME', '') + self.environ['PATH_INFO']) @property def swift_entity_path(self): """ Provides the account/container/object path, sans API version. This can be useful when constructing a path to send to a backend server, as that path will need everything after the "/v1". """ _ver, entity_path = self.split_path(1, 2, rest_with_last=True) if entity_path is not None: return '/' + entity_path @property def url(self): "Provides the full url of the request" return self.host_url + self.path_qs def as_referer(self): return self.method + ' ' + self.url def path_info_pop(self): """ Takes one path portion (delineated by slashes) from the path_info, and appends it to the script_name. Returns the path segment. """ path_info = self.path_info if not path_info or path_info[0] != '/': return None try: slash_loc = path_info.index('/', 1) except ValueError: slash_loc = len(path_info) self.script_name += path_info[:slash_loc] self.path_info = path_info[slash_loc:] return path_info[1:slash_loc] def copy_get(self): """ Makes a copy of the request, converting it to a GET. """ env = self.environ.copy() env.update({ 'REQUEST_METHOD': 'GET', 'CONTENT_LENGTH': '0', 'wsgi.input': StringIO(''), }) return Request(env) def call_application(self, application): """ Calls the application with this request's environment. Returns the status, headers, and app_iter for the response as a tuple. :param application: the WSGI application to call """ output = [] captured = [] def start_response(status, headers, exc_info=None): captured[:] = [status, headers, exc_info] return output.append app_iter = application(self.environ, start_response) if not app_iter: app_iter = output if not captured: app_iter = reiterate(app_iter) return (captured[0], captured[1], app_iter) def get_response(self, application): """ Calls the application with this request's environment. Returns a Response object that wraps up the application's result. :param application: the WSGI application to call """ status, headers, app_iter = self.call_application(application) return Response(status=status, headers=dict(headers), app_iter=app_iter, request=self) def split_path(self, minsegs=1, maxsegs=None, rest_with_last=False): """ Validate and split the Request's path. **Examples**:: ['a'] = split_path('/a') ['a', None] = split_path('/a', 1, 2) ['a', 'c'] = split_path('/a/c', 1, 2) ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True) :param minsegs: Minimum number of segments to be extracted :param maxsegs: Maximum number of segments to be extracted :param rest_with_last: If True, trailing data will be returned as part of last segment. If False, and there is trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) :raises: ValueError if given an invalid path """ return split_path( self.environ.get('SCRIPT_NAME', '') + self.environ['PATH_INFO'], minsegs, maxsegs, rest_with_last) def message_length(self): """ Properly determine the message length for this request. It will return an integer if the headers explicitly contain the message length, or None if the headers don't contain a length. The ValueError exception will be raised if the headers are invalid. :raises ValueError: if either transfer-encoding or content-length headers have bad values :raises AttributeError: if the last value of the transfer-encoding header is not "chunked" """ te = self.headers.get('transfer-encoding') if te: encodings = te.split(',') if len(encodings) > 1: raise AttributeError('Unsupported Transfer-Coding header' ' value specified in Transfer-Encoding' ' header') # If there are more than one transfer encoding value, the last # one must be chunked, see RFC 2616 Sec. 3.6 if encodings[-1].lower() == 'chunked': chunked = True else: raise ValueError('Invalid Transfer-Encoding header value') else: chunked = False if not chunked: # Because we are not using chunked transfer encoding we can pay # attention to the content-length header. fsize = self.headers.get('content-length', None) if fsize is not None: try: fsize = int(fsize) except ValueError: raise ValueError('Invalid Content-Length header value') else: fsize = None return fsize def content_range_header_value(start, stop, size): return 'bytes %s-%s/%s' % (start, (stop - 1), size) def content_range_header(start, stop, size): return "Content-Range: " + content_range_header_value(start, stop, size) def multi_range_iterator(ranges, content_type, boundary, size, sub_iter_gen): for start, stop in ranges: yield ''.join(['\r\n--', boundary, '\r\n', 'Content-Type: ', content_type, '\r\n']) yield content_range_header(start, stop, size) + '\r\n\r\n' sub_iter = sub_iter_gen(start, stop) for chunk in sub_iter: yield chunk yield '\r\n--' + boundary + '--\r\n' class Response(object): """ WSGI Response object. """ content_length = _header_int_property('content-length') content_type = _resp_content_type_property() content_range = _header_property('content-range') etag = _resp_etag_property() status = _resp_status_property() body = _resp_body_property() host_url = _host_url_property() last_modified = _datetime_property('last-modified') location = _header_property('location') accept_ranges = _header_property('accept-ranges') charset = _resp_charset_property() app_iter = _resp_app_iter_property() def __init__(self, body=None, status=200, headers=None, app_iter=None, request=None, conditional_response=False, **kw): self.headers = HeaderKeyDict( [('Content-Type', 'text/html; charset=UTF-8')]) self.conditional_response = conditional_response self.request = request self.body = body self.app_iter = app_iter self.status = status self.boundary = "%.32x" % random.randint(0, 256 ** 16) if request: self.environ = request.environ else: self.environ = {} if headers: self.headers.update(headers) if self.status_int == 401 and 'www-authenticate' not in self.headers: self.headers.update({'www-authenticate': self.www_authenticate()}) for key, value in kw.iteritems(): setattr(self, key, value) # When specifying both 'content_type' and 'charset' in the kwargs, # charset needs to be applied *after* content_type, otherwise charset # can get wiped out when content_type sorts later in dict order. if 'charset' in kw and 'content_type' in kw: self.charset = kw['charset'] def _prepare_for_ranges(self, ranges): """ Prepare the Response for multiple ranges. """ content_size = self.content_length content_type = self.content_type self.content_type = ''.join(['multipart/byteranges;', 'boundary=', self.boundary]) # This section calculate the total size of the targeted response # The value 12 is the length of total bytes of hyphen, new line # form feed for each section header. The value 8 is the length of # total bytes of hyphen, new line, form feed characters for the # closing boundary which appears only once section_header_fixed_len = 12 + (len(self.boundary) + len('Content-Type: ') + len(content_type) + len('Content-Range: bytes ')) body_size = 0 for start, end in ranges: body_size += section_header_fixed_len body_size += len(str(start) + '-' + str(end - 1) + '/' + str(content_size)) + (end - start) body_size += 8 + len(self.boundary) self.content_length = body_size self.content_range = None return content_size, content_type def _response_iter(self, app_iter, body): if self.conditional_response and self.request: if self.etag and self.request.if_none_match and \ self.etag in self.request.if_none_match: self.status = 304 self.content_length = 0 return [''] if self.etag and self.request.if_match and \ self.etag not in self.request.if_match: self.status = 412 self.content_length = 0 return [''] if self.status_int == 404 and self.request.if_match \ and '*' in self.request.if_match: # If none of the entity tags match, or if "*" is given and no # current entity exists, the server MUST NOT perform the # requested method, and MUST return a 412 (Precondition # Failed) response. [RFC 2616 section 14.24] self.status = 412 self.content_length = 0 return [''] if self.request and self.request.method == 'HEAD': # We explicitly do NOT want to set self.content_length to 0 here return [''] if self.conditional_response and self.request and \ self.request.range and self.request.range.ranges and \ not self.content_range: ranges = self.request.range.ranges_for_length(self.content_length) if ranges == []: self.status = 416 self.content_length = 0 return [''] elif ranges: range_size = len(ranges) if range_size > 0: # There is at least one valid range in the request, so try # to satisfy the request if range_size == 1: start, end = ranges[0] if app_iter and hasattr(app_iter, 'app_iter_range'): self.status = 206 self.content_range = content_range_header_value( start, end, self.content_length) self.content_length = (end - start) return app_iter.app_iter_range(start, end) elif body: self.status = 206 self.content_range = content_range_header_value( start, end, self.content_length) self.content_length = (end - start) return [body[start:end]] elif range_size > 1: if app_iter and hasattr(app_iter, 'app_iter_ranges'): self.status = 206 content_size, content_type = \ self._prepare_for_ranges(ranges) return app_iter.app_iter_ranges(ranges, content_type, self.boundary, content_size) elif body: self.status = 206 content_size, content_type, = \ self._prepare_for_ranges(ranges) def _body_slicer(start, stop): yield body[start:stop] return multi_range_iterator(ranges, content_type, self.boundary, content_size, _body_slicer) if app_iter: return app_iter if body is not None: return [body] if self.status_int in RESPONSE_REASONS: title, exp = RESPONSE_REASONS[self.status_int] if exp: body = '

%s

%s

' % (title, exp) if '%(' in body: body = body % defaultdict(lambda: 'unknown', self.__dict__) self.content_length = len(body) return [body] return [''] def absolute_location(self): """ Attempt to construct an absolute location. """ if not self.location.startswith('/'): return self.location return self.host_url + self.location def www_authenticate(self): """ Construct a suitable value for WWW-Authenticate response header If we have a request and a valid-looking path, the realm is the account; otherwise we set it to 'unknown'. """ try: vrs, realm, rest = self.request.split_path(2, 3, True) if realm in ('v1.0', 'auth'): realm = 'unknown' except (AttributeError, ValueError): realm = 'unknown' return 'Swift realm="%s"' % realm @property def is_success(self): return self.status_int // 100 == 2 def __call__(self, env, start_response): """ Respond to the WSGI request. .. warning:: This will translate any relative Location header value to an absolute URL using the WSGI environment's HOST_URL as a prefix, as RFC 2616 specifies. However, it is quite common to use relative redirects, especially when it is difficult to know the exact HOST_URL the browser would have used when behind several CNAMEs, CDN services, etc. All modern browsers support relative redirects. To skip over RFC enforcement of the Location header value, you may set ``env['swift.leave_relative_location'] = True`` in the WSGI environment. """ if not self.request: self.request = Request(env) self.environ = env app_iter = self._response_iter(self.app_iter, self._body) if 'location' in self.headers and \ not env.get('swift.leave_relative_location'): self.location = self.absolute_location() start_response(self.status, self.headers.items()) return app_iter class HTTPException(Response, Exception): def __init__(self, *args, **kwargs): Response.__init__(self, *args, **kwargs) Exception.__init__(self, self.status) def wsgify(func): """ A decorator for translating functions which take a swob Request object and return a Response object into WSGI callables. Also catches any raised HTTPExceptions and treats them as a returned Response. """ argspec = inspect.getargspec(func) if argspec.args and argspec.args[0] == 'self': @functools.wraps(func) def _wsgify_self(self, env, start_response): try: return func(self, Request(env))(env, start_response) except HTTPException as err_resp: return err_resp(env, start_response) return _wsgify_self else: @functools.wraps(func) def _wsgify_bare(env, start_response): try: return func(Request(env))(env, start_response) except HTTPException as err_resp: return err_resp(env, start_response) return _wsgify_bare class StatusMap(object): """ A dict-like object that returns HTTPException subclasses/factory functions where the given key is the status code. """ def __getitem__(self, key): return partial(HTTPException, status=key) status_map = StatusMap() HTTPOk = status_map[200] HTTPCreated = status_map[201] HTTPAccepted = status_map[202] HTTPNoContent = status_map[204] HTTPMovedPermanently = status_map[301] HTTPFound = status_map[302] HTTPSeeOther = status_map[303] HTTPNotModified = status_map[304] HTTPTemporaryRedirect = status_map[307] HTTPBadRequest = status_map[400] HTTPUnauthorized = status_map[401] HTTPForbidden = status_map[403] HTTPMethodNotAllowed = status_map[405] HTTPNotFound = status_map[404] HTTPNotAcceptable = status_map[406] HTTPRequestTimeout = status_map[408] HTTPConflict = status_map[409] HTTPLengthRequired = status_map[411] HTTPPreconditionFailed = status_map[412] HTTPRequestEntityTooLarge = status_map[413] HTTPRequestedRangeNotSatisfiable = status_map[416] HTTPUnprocessableEntity = status_map[422] HTTPClientDisconnect = status_map[499] HTTPServerError = status_map[500] HTTPInternalServerError = status_map[500] HTTPNotImplemented = status_map[501] HTTPBadGateway = status_map[502] HTTPServiceUnavailable = status_map[503] HTTPInsufficientStorage = status_map[507] swift-1.13.1/swift/common/utils.py0000664000175400017540000025451712323703614020244 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" import errno import fcntl import grp import hmac import operator import os import pwd import re import sys import threading as stdlib_threading import time import uuid import functools import weakref from hashlib import md5, sha1 from random import random, shuffle from urllib import quote as _quote from contextlib import contextmanager, closing import ctypes import ctypes.util from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \ RawConfigParser from optparse import OptionParser from Queue import Queue, Empty from tempfile import mkstemp, NamedTemporaryFile try: import simplejson as json except ImportError: import json import cPickle as pickle import glob from urlparse import urlparse as stdlib_urlparse, ParseResult import itertools import stat import eventlet import eventlet.semaphore from eventlet import GreenPool, sleep, Timeout, tpool, greenthread, \ greenio, event from eventlet.green import socket, threading import eventlet.queue import netifaces import codecs utf8_decoder = codecs.getdecoder('utf-8') utf8_encoder = codecs.getencoder('utf-8') from swift import gettext_ as _ from swift.common.exceptions import LockTimeout, MessageTimeout from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND # logging doesn't import patched as cleanly as one would like from logging.handlers import SysLogHandler import logging logging.thread = eventlet.green.thread logging.threading = eventlet.green.threading logging._lock = logging.threading.RLock() # setup notice level logging NOTICE = 25 logging._levelNames[NOTICE] = 'NOTICE' SysLogHandler.priority_map['NOTICE'] = 'notice' # These are lazily pulled from libc elsewhere _sys_fallocate = None _posix_fadvise = None # If set to non-zero, fallocate routines will fail based on free space # available being at or below this amount, in bytes. FALLOCATE_RESERVE = 0 # Used by hash_path to offer a bit more security when generating hashes for # paths. It simply appends this value to all paths; guessing the hash a path # will end up with would also require knowing this suffix. HASH_PATH_SUFFIX = '' HASH_PATH_PREFIX = '' SWIFT_CONF_FILE = '/etc/swift/swift.conf' class InvalidHashPathConfigError(ValueError): def __str__(self): return "[swift-hash]: both swift_hash_path_suffix and " \ "swift_hash_path_prefix are missing from %s" % SWIFT_CONF_FILE def validate_hash_conf(): global HASH_PATH_SUFFIX global HASH_PATH_PREFIX if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: hash_conf = ConfigParser() if hash_conf.read(SWIFT_CONF_FILE): try: HASH_PATH_SUFFIX = hash_conf.get('swift-hash', 'swift_hash_path_suffix') except (NoSectionError, NoOptionError): pass try: HASH_PATH_PREFIX = hash_conf.get('swift-hash', 'swift_hash_path_prefix') except (NoSectionError, NoOptionError): pass if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX: raise InvalidHashPathConfigError() try: validate_hash_conf() except InvalidHashPathConfigError: # could get monkey patched or lazy loaded pass def get_hmac(request_method, path, expires, key): """ Returns the hexdigest string of the HMAC-SHA1 (RFC 2104) for the request. :param request_method: Request method to allow. :param path: The path to the resource to allow access to. :param expires: Unix timestamp as an int for when the URL expires. :param key: HMAC shared secret. :returns: hexdigest str of the HMAC-SHA1 for the request. """ return hmac.new( key, '%s\n%s\n%s' % (request_method, expires, path), sha1).hexdigest() # Used by get_swift_info and register_swift_info to store information about # the swift cluster. _swift_info = {} _swift_admin_info = {} def get_swift_info(admin=False, disallowed_sections=None): """ Returns information about the swift cluster that has been previously registered with the register_swift_info call. :param admin: boolean value, if True will additionally return an 'admin' section with information previously registered as admin info. :param disallowed_sections: list of section names to be withheld from the information returned. :returns: dictionary of information about the swift cluster. """ disallowed_sections = disallowed_sections or [] info = {} for section in _swift_info: if section in disallowed_sections: continue info[section] = dict(_swift_info[section].items()) if admin: info['admin'] = dict(_swift_admin_info) info['admin']['disallowed_sections'] = list(disallowed_sections) return info def register_swift_info(name='swift', admin=False, **kwargs): """ Registers information about the swift cluster to be retrieved with calls to get_swift_info. :param name: string, the section name to place the information under. :param admin: boolean, if True, information will be registered to an admin section which can optionally be withheld when requesting the information. :param kwargs: key value arguments representing the information to be added. """ if name == 'admin' or name == 'disallowed_sections': raise ValueError('\'{0}\' is reserved name.'.format(name)) if admin: dict_to_use = _swift_admin_info else: dict_to_use = _swift_info if name not in dict_to_use: dict_to_use[name] = {} for key, val in kwargs.iteritems(): dict_to_use[name][key] = val def backward(f, blocksize=4096): """ A generator returning lines from a file starting with the last line, then the second last line, etc. i.e., it reads lines backwards. Stops when the first line (if any) is read. This is useful when searching for recent activity in very large files. :param f: file object to read :param blocksize: no of characters to go backwards at each block """ f.seek(0, os.SEEK_END) if f.tell() == 0: return last_row = '' while f.tell() != 0: try: f.seek(-blocksize, os.SEEK_CUR) except IOError: blocksize = f.tell() f.seek(-blocksize, os.SEEK_CUR) block = f.read(blocksize) f.seek(-blocksize, os.SEEK_CUR) rows = block.split('\n') rows[-1] = rows[-1] + last_row while rows: last_row = rows.pop(-1) if rows and last_row: yield last_row yield last_row # Used when reading config values TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) def config_true_value(value): """ Returns True if the value is either True or a string in TRUE_VALUES. Returns False otherwise. """ return value is True or \ (isinstance(value, basestring) and value.lower() in TRUE_VALUES) def config_auto_int_value(value, default): """ Returns default if value is None or 'auto'. Returns value as an int or raises ValueError otherwise. """ if value is None or \ (isinstance(value, basestring) and value.lower() == 'auto'): return default try: value = int(value) except (TypeError, ValueError): raise ValueError('Config option must be a integer or the ' 'string "auto", not "%s".' % value) return value def noop_libc_function(*args): return 0 def validate_configuration(): try: validate_hash_conf() except InvalidHashPathConfigError as e: sys.exit("Error: %s" % e) def load_libc_function(func_name, log_error=True): """ Attempt to find the function in libc, otherwise return a no-op func. :param func_name: name of the function to pull from libc. """ try: libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) return getattr(libc, func_name) except AttributeError: if log_error: logging.warn(_("Unable to locate %s in libc. Leaving as a " "no-op."), func_name) return noop_libc_function def generate_trans_id(trans_id_suffix): return 'tx%s-%010x%s' % ( uuid.uuid4().hex[:21], time.time(), trans_id_suffix) def get_trans_id_time(trans_id): if len(trans_id) >= 34 and trans_id[:2] == 'tx' and trans_id[23] == '-': try: return int(trans_id[24:34], 16) except ValueError: pass return None class FileLikeIter(object): def __init__(self, iterable): """ Wraps an iterable to behave as a file-like object. """ self.iterator = iter(iterable) self.buf = None self.closed = False def __iter__(self): return self def next(self): """ x.next() -> the next value, or raise StopIteration """ if self.closed: raise ValueError('I/O operation on closed file') if self.buf: rv = self.buf self.buf = None return rv else: return self.iterator.next() def read(self, size=-1): """ read([size]) -> read at most size bytes, returned as a string. If the size argument is negative or omitted, read until EOF is reached. Notice that when in non-blocking mode, less data than what was requested may be returned, even if no size parameter was given. """ if self.closed: raise ValueError('I/O operation on closed file') if size < 0: return ''.join(self) elif not size: chunk = '' elif self.buf: chunk = self.buf self.buf = None else: try: chunk = self.iterator.next() except StopIteration: return '' if len(chunk) > size: self.buf = chunk[size:] chunk = chunk[:size] return chunk def readline(self, size=-1): """ readline([size]) -> next line from the file, as a string. Retain newline. A non-negative size argument limits the maximum number of bytes to return (an incomplete line may be returned then). Return an empty string at EOF. """ if self.closed: raise ValueError('I/O operation on closed file') data = '' while '\n' not in data and (size < 0 or len(data) < size): if size < 0: chunk = self.read(1024) else: chunk = self.read(size - len(data)) if not chunk: break data += chunk if '\n' in data: data, sep, rest = data.partition('\n') data += sep if self.buf: self.buf = rest + self.buf else: self.buf = rest return data def readlines(self, sizehint=-1): """ readlines([size]) -> list of strings, each a line from the file. Call readline() repeatedly and return a list of the lines so read. The optional size argument, if given, is an approximate bound on the total number of bytes in the lines returned. """ if self.closed: raise ValueError('I/O operation on closed file') lines = [] while True: line = self.readline(sizehint) if not line: break lines.append(line) if sizehint >= 0: sizehint -= len(line) if sizehint <= 0: break return lines def close(self): """ close() -> None or (perhaps) an integer. Close the file. Sets data attribute .closed to True. A closed file cannot be used for further I/O operations. close() may be called more than once without error. Some kinds of file objects (for example, opened by popen()) may return an exit status upon closing. """ self.iterator = None self.closed = True class FallocateWrapper(object): def __init__(self, noop=False): if noop: self.func_name = 'posix_fallocate' self.fallocate = noop_libc_function return ## fallocate is preferred because we need the on-disk size to match ## the allocated size. Older versions of sqlite require that the ## two sizes match. However, fallocate is Linux only. for func in ('fallocate', 'posix_fallocate'): self.func_name = func self.fallocate = load_libc_function(func, log_error=False) if self.fallocate is not noop_libc_function: break if self.fallocate is noop_libc_function: logging.warn(_("Unable to locate fallocate, posix_fallocate in " "libc. Leaving as a no-op.")) def __call__(self, fd, mode, offset, length): """The length parameter must be a ctypes.c_uint64.""" if FALLOCATE_RESERVE > 0: st = os.fstatvfs(fd) free = st.f_frsize * st.f_bavail - length.value if free <= FALLOCATE_RESERVE: raise OSError('FALLOCATE_RESERVE fail %s <= %s' % ( free, FALLOCATE_RESERVE)) args = { 'fallocate': (fd, mode, offset, length), 'posix_fallocate': (fd, offset, length) } return self.fallocate(*args[self.func_name]) def disable_fallocate(): global _sys_fallocate _sys_fallocate = FallocateWrapper(noop=True) def fallocate(fd, size): """ Pre-allocate disk space for a file. :param fd: file descriptor :param size: size to allocate (in bytes) """ global _sys_fallocate if _sys_fallocate is None: _sys_fallocate = FallocateWrapper() if size < 0: size = 0 # 1 means "FALLOC_FL_KEEP_SIZE", which means it pre-allocates invisibly ret = _sys_fallocate(fd, 1, 0, ctypes.c_uint64(size)) err = ctypes.get_errno() if ret and err not in (0, errno.ENOSYS, errno.EOPNOTSUPP, errno.EINVAL): raise OSError(err, 'Unable to fallocate(%s)' % size) def fsync(fd): """ Sync modified file data and metadata to disk. :param fd: file descriptor """ if hasattr(fcntl, 'F_FULLSYNC'): try: fcntl.fcntl(fd, fcntl.F_FULLSYNC) except IOError as e: raise OSError(e.errno, 'Unable to F_FULLSYNC(%s)' % fd) else: os.fsync(fd) def fdatasync(fd): """ Sync modified file data to disk. :param fd: file descriptor """ try: os.fdatasync(fd) except AttributeError: fsync(fd) def drop_buffer_cache(fd, offset, length): """ Drop 'buffer' cache for the given range of the given file. :param fd: file descriptor :param offset: start offset :param length: length """ global _posix_fadvise if _posix_fadvise is None: _posix_fadvise = load_libc_function('posix_fadvise64') # 4 means "POSIX_FADV_DONTNEED" ret = _posix_fadvise(fd, ctypes.c_uint64(offset), ctypes.c_uint64(length), 4) if ret != 0: logging.warn("posix_fadvise64(%s, %s, %s, 4) -> %s" % (fd, offset, length, ret)) def normalize_timestamp(timestamp): """ Format a timestamp (string or numeric) into a standardized xxxxxxxxxx.xxxxx (10.5) format. Note that timestamps using values greater than or equal to November 20th, 2286 at 17:46 UTC will use 11 digits to represent the number of seconds. :param timestamp: unix timestamp :returns: normalized timestamp as a string """ return "%016.05f" % (float(timestamp)) def normalize_delete_at_timestamp(timestamp): """ Format a timestamp (string or numeric) into a standardized xxxxxxxxxx (10) format. Note that timestamps less than 0000000000 are raised to 0000000000 and values greater than November 20th, 2286 at 17:46:39 UTC will be capped at that date and time, resulting in no return value exceeding 9999999999. This cap is because the expirer is already working through a sorted list of strings that were all a length of 10. Adding another digit would mess up the sort and cause the expirer to break from processing early. By 2286, this problem will need to be fixed, probably by creating an additional .expiring_objects account to work from with 11 (or more) digit container names. :param timestamp: unix timestamp :returns: normalized timestamp as a string """ return '%010d' % min(max(0, float(timestamp)), 9999999999) def mkdirs(path): """ Ensures the path is a directory or makes it if not. Errors if the path exists but is a file or on permissions failure. :param path: path to create """ if not os.path.isdir(path): try: os.makedirs(path) except OSError as err: if err.errno != errno.EEXIST or not os.path.isdir(path): raise def renamer(old, new): """ Attempt to fix / hide race conditions like empty object directories being removed by backend processes during uploads, by retrying. :param old: old path to be renamed :param new: new path to be renamed to """ try: mkdirs(os.path.dirname(new)) os.rename(old, new) except OSError: mkdirs(os.path.dirname(new)) os.rename(old, new) def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): """ Validate and split the given HTTP request path. **Examples**:: ['a'] = split_path('/a') ['a', None] = split_path('/a', 1, 2) ['a', 'c'] = split_path('/a/c', 1, 2) ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True) :param path: HTTP Request path to be split :param minsegs: Minimum number of segments to be extracted :param maxsegs: Maximum number of segments to be extracted :param rest_with_last: If True, trailing data will be returned as part of last segment. If False, and there is trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) :raises: ValueError if given an invalid path """ if not maxsegs: maxsegs = minsegs if minsegs > maxsegs: raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) if rest_with_last: segs = path.split('/', maxsegs) minsegs += 1 maxsegs += 1 count = len(segs) if (segs[0] or count < minsegs or count > maxsegs or '' in segs[1:minsegs]): raise ValueError('Invalid path: %s' % quote(path)) else: minsegs += 1 maxsegs += 1 segs = path.split('/', maxsegs) count = len(segs) if (segs[0] or count < minsegs or count > maxsegs + 1 or '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs])): raise ValueError('Invalid path: %s' % quote(path)) segs = segs[1:maxsegs] segs.extend([None] * (maxsegs - 1 - len(segs))) return segs def validate_device_partition(device, partition): """ Validate that a device and a partition are valid and won't lead to directory traversal when used. :param device: device to validate :param partition: partition to validate :raises: ValueError if given an invalid device or partition """ invalid_device = False invalid_partition = False if not device or '/' in device or device in ['.', '..']: invalid_device = True if not partition or '/' in partition or partition in ['.', '..']: invalid_partition = True if invalid_device: raise ValueError('Invalid device: %s' % quote(device or '')) elif invalid_partition: raise ValueError('Invalid partition: %s' % quote(partition or '')) class RateLimitedIterator(object): """ Wrap an iterator to only yield elements at a rate of N per second. :param iterable: iterable to wrap :param elements_per_second: the rate at which to yield elements :param limit_after: rate limiting kicks in only after yielding this many elements; default is 0 (rate limit immediately) """ def __init__(self, iterable, elements_per_second, limit_after=0): self.iterator = iter(iterable) self.elements_per_second = elements_per_second self.limit_after = limit_after self.running_time = 0 def __iter__(self): return self def next(self): if self.limit_after > 0: self.limit_after -= 1 else: self.running_time = ratelimit_sleep(self.running_time, self.elements_per_second) return self.iterator.next() class GreenthreadSafeIterator(object): """ Wrap an iterator to ensure that only one greenthread is inside its next() method at a time. This is useful if an iterator's next() method may perform network IO, as that may trigger a greenthread context switch (aka trampoline), which can give another greenthread a chance to call next(). At that point, you get an error like "ValueError: generator already executing". By wrapping calls to next() with a mutex, we avoid that error. """ def __init__(self, unsafe_iterable): self.unsafe_iter = iter(unsafe_iterable) self.semaphore = eventlet.semaphore.Semaphore(value=1) def __iter__(self): return self def next(self): with self.semaphore: return self.unsafe_iter.next() class NullLogger(object): """A no-op logger for eventlet wsgi.""" def write(self, *args): #"Logs" the args to nowhere pass class LoggerFileObject(object): def __init__(self, logger): self.logger = logger def write(self, value): value = value.strip() if value: if 'Connection reset by peer' in value: self.logger.error(_('STDOUT: Connection reset by peer')) else: self.logger.error(_('STDOUT: %s'), value) def writelines(self, values): self.logger.error(_('STDOUT: %s'), '#012'.join(values)) def close(self): pass def flush(self): pass def __iter__(self): return self def next(self): raise IOError(errno.EBADF, 'Bad file descriptor') def read(self, size=-1): raise IOError(errno.EBADF, 'Bad file descriptor') def readline(self, size=-1): raise IOError(errno.EBADF, 'Bad file descriptor') def tell(self): return 0 def xreadlines(self): return self class StatsdClient(object): def __init__(self, host, port, base_prefix='', tail_prefix='', default_sample_rate=1, sample_rate_factor=1): self._host = host self._port = port self._base_prefix = base_prefix self.set_prefix(tail_prefix) self._default_sample_rate = default_sample_rate self._sample_rate_factor = sample_rate_factor self._target = (self._host, self._port) self.random = random def set_prefix(self, new_prefix): if new_prefix and self._base_prefix: self._prefix = '.'.join([self._base_prefix, new_prefix, '']) elif new_prefix: self._prefix = new_prefix + '.' elif self._base_prefix: self._prefix = self._base_prefix + '.' else: self._prefix = '' def _send(self, m_name, m_value, m_type, sample_rate): if sample_rate is None: sample_rate = self._default_sample_rate sample_rate = sample_rate * self._sample_rate_factor parts = ['%s%s:%s' % (self._prefix, m_name, m_value), m_type] if sample_rate < 1: if self.random() < sample_rate: parts.append('@%s' % (sample_rate,)) else: return # Ideally, we'd cache a sending socket in self, but that # results in a socket getting shared by multiple green threads. with closing(self._open_socket()) as sock: return sock.sendto('|'.join(parts), self._target) def _open_socket(self): return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) def update_stats(self, m_name, m_value, sample_rate=None): return self._send(m_name, m_value, 'c', sample_rate) def increment(self, metric, sample_rate=None): return self.update_stats(metric, 1, sample_rate) def decrement(self, metric, sample_rate=None): return self.update_stats(metric, -1, sample_rate) def timing(self, metric, timing_ms, sample_rate=None): return self._send(metric, timing_ms, 'ms', sample_rate) def timing_since(self, metric, orig_time, sample_rate=None): return self.timing(metric, (time.time() - orig_time) * 1000, sample_rate) def transfer_rate(self, metric, elapsed_time, byte_xfer, sample_rate=None): if byte_xfer: return self.timing(metric, elapsed_time * 1000 / byte_xfer * 1000, sample_rate) def timing_stats(**dec_kwargs): """ Returns a decorator that logs timing events or errors for public methods in swift's wsgi server controllers, based on response code. """ def decorating_func(func): method = func.func_name @functools.wraps(func) def _timing_stats(ctrl, *args, **kwargs): start_time = time.time() resp = func(ctrl, *args, **kwargs) if is_success(resp.status_int) or \ is_redirection(resp.status_int) or \ resp.status_int == HTTP_NOT_FOUND: ctrl.logger.timing_since(method + '.timing', start_time, **dec_kwargs) else: ctrl.logger.timing_since(method + '.errors.timing', start_time, **dec_kwargs) return resp return _timing_stats return decorating_func class LoggingHandlerWeakRef(weakref.ref): """ Like a weak reference, but passes through a couple methods that logging handlers need. """ def close(self): referent = self() try: if referent: referent.close() except KeyError: # This is to catch an issue with old py2.6 versions pass def flush(self): referent = self() if referent: referent.flush() # double inheritance to support property with setter class LogAdapter(logging.LoggerAdapter, object): """ A Logger like object which performs some reformatting on calls to :meth:`exception`. Can be used to store a threadlocal transaction id and client ip. """ _cls_thread_local = threading.local() def __init__(self, logger, server): logging.LoggerAdapter.__init__(self, logger, {}) self.server = server setattr(self, 'warn', self.warning) @property def txn_id(self): if hasattr(self._cls_thread_local, 'txn_id'): return self._cls_thread_local.txn_id @txn_id.setter def txn_id(self, value): self._cls_thread_local.txn_id = value @property def client_ip(self): if hasattr(self._cls_thread_local, 'client_ip'): return self._cls_thread_local.client_ip @client_ip.setter def client_ip(self, value): self._cls_thread_local.client_ip = value @property def thread_locals(self): return (self.txn_id, self.client_ip) @thread_locals.setter def thread_locals(self, value): self.txn_id, self.client_ip = value def getEffectiveLevel(self): return self.logger.getEffectiveLevel() def process(self, msg, kwargs): """ Add extra info to message """ kwargs['extra'] = {'server': self.server, 'txn_id': self.txn_id, 'client_ip': self.client_ip} return msg, kwargs def notice(self, msg, *args, **kwargs): """ Convenience function for syslog priority LOG_NOTICE. The python logging lvl is set to 25, just above info. SysLogHandler is monkey patched to map this log lvl to the LOG_NOTICE syslog priority. """ self.log(NOTICE, msg, *args, **kwargs) def _exception(self, msg, *args, **kwargs): logging.LoggerAdapter.exception(self, msg, *args, **kwargs) def exception(self, msg, *args, **kwargs): _junk, exc, _junk = sys.exc_info() call = self.error emsg = '' if isinstance(exc, OSError): if exc.errno in (errno.EIO, errno.ENOSPC): emsg = str(exc) else: call = self._exception elif isinstance(exc, socket.error): if exc.errno == errno.ECONNREFUSED: emsg = _('Connection refused') elif exc.errno == errno.EHOSTUNREACH: emsg = _('Host unreachable') elif exc.errno == errno.ETIMEDOUT: emsg = _('Connection timeout') else: call = self._exception elif isinstance(exc, eventlet.Timeout): emsg = exc.__class__.__name__ if hasattr(exc, 'seconds'): emsg += ' (%ss)' % exc.seconds if isinstance(exc, MessageTimeout): if exc.msg: emsg += ' %s' % exc.msg else: call = self._exception call('%s: %s' % (msg, emsg), *args, **kwargs) def set_statsd_prefix(self, prefix): """ The StatsD client prefix defaults to the "name" of the logger. This method may override that default with a specific value. Currently used in the proxy-server to differentiate the Account, Container, and Object controllers. """ if self.logger.statsd_client: self.logger.statsd_client.set_prefix(prefix) def statsd_delegate(statsd_func_name): """ Factory to create methods which delegate to methods on self.logger.statsd_client (an instance of StatsdClient). The created methods conditionally delegate to a method whose name is given in 'statsd_func_name'. The created delegate methods are a no-op when StatsD logging is not configured. :param statsd_func_name: the name of a method on StatsdClient. """ func = getattr(StatsdClient, statsd_func_name) @functools.wraps(func) def wrapped(self, *a, **kw): if getattr(self.logger, 'statsd_client'): return func(self.logger.statsd_client, *a, **kw) return wrapped update_stats = statsd_delegate('update_stats') increment = statsd_delegate('increment') decrement = statsd_delegate('decrement') timing = statsd_delegate('timing') timing_since = statsd_delegate('timing_since') transfer_rate = statsd_delegate('transfer_rate') class SwiftLogFormatter(logging.Formatter): """ Custom logging.Formatter will append txn_id to a log message if the record has one and the message does not. """ def format(self, record): if not hasattr(record, 'server'): # Catch log messages that were not initiated by swift # (for example, the keystone auth middleware) record.server = record.name # Included from Python's logging.Formatter and then altered slightly to # replace \n with #012 record.message = record.getMessage() if self._fmt.find('%(asctime)') >= 0: record.asctime = self.formatTime(record, self.datefmt) msg = (self._fmt % record.__dict__).replace('\n', '#012') if record.exc_info: # Cache the traceback text to avoid converting it multiple times # (it's constant anyway) if not record.exc_text: record.exc_text = self.formatException( record.exc_info).replace('\n', '#012') if record.exc_text: if msg[-3:] != '#012': msg = msg + '#012' msg = msg + record.exc_text if (hasattr(record, 'txn_id') and record.txn_id and record.levelno != logging.INFO and record.txn_id not in msg): msg = "%s (txn: %s)" % (msg, record.txn_id) if (hasattr(record, 'client_ip') and record.client_ip and record.levelno != logging.INFO and record.client_ip not in msg): msg = "%s (client_ip: %s)" % (msg, record.client_ip) return msg def get_logger(conf, name=None, log_to_console=False, log_route=None, fmt="%(server)s: %(message)s"): """ Get the current system logger using config settings. **Log config and defaults**:: log_facility = LOG_LOCAL0 log_level = INFO log_name = swift log_udp_host = (disabled) log_udp_port = logging.handlers.SYSLOG_UDP_PORT log_address = /dev/log log_statsd_host = (disabled) log_statsd_port = 8125 log_statsd_default_sample_rate = 1.0 log_statsd_sample_rate_factor = 1.0 log_statsd_metric_prefix = (empty-string) :param conf: Configuration dict to read settings from :param name: Name of the logger :param log_to_console: Add handler which writes to console on stderr :param log_route: Route for the logging, not emitted to the log, just used to separate logging configurations :param fmt: Override log format """ if not conf: conf = {} if name is None: name = conf.get('log_name', 'swift') if not log_route: log_route = name logger = logging.getLogger(log_route) logger.propagate = False # all new handlers will get the same formatter formatter = SwiftLogFormatter(fmt) # get_logger will only ever add one SysLog Handler to a logger if not hasattr(get_logger, 'handler4logger'): get_logger.handler4logger = {} if logger in get_logger.handler4logger: logger.removeHandler(get_logger.handler4logger[logger]) # facility for this logger will be set by last call wins facility = getattr(SysLogHandler, conf.get('log_facility', 'LOG_LOCAL0'), SysLogHandler.LOG_LOCAL0) udp_host = conf.get('log_udp_host') if udp_host: udp_port = int(conf.get('log_udp_port', logging.handlers.SYSLOG_UDP_PORT)) handler = SysLogHandler(address=(udp_host, udp_port), facility=facility) else: log_address = conf.get('log_address', '/dev/log') try: handler = SysLogHandler(address=log_address, facility=facility) except socket.error as e: # Either /dev/log isn't a UNIX socket or it does not exist at all if e.errno not in [errno.ENOTSOCK, errno.ENOENT]: raise e handler = SysLogHandler(facility=facility) handler.setFormatter(formatter) logger.addHandler(handler) get_logger.handler4logger[logger] = handler # setup console logging if log_to_console or hasattr(get_logger, 'console_handler4logger'): # remove pre-existing console handler for this logger if not hasattr(get_logger, 'console_handler4logger'): get_logger.console_handler4logger = {} if logger in get_logger.console_handler4logger: logger.removeHandler(get_logger.console_handler4logger[logger]) console_handler = logging.StreamHandler(sys.__stderr__) console_handler.setFormatter(formatter) logger.addHandler(console_handler) get_logger.console_handler4logger[logger] = console_handler # set the level for the logger logger.setLevel( getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) # Setup logger with a StatsD client if so configured statsd_host = conf.get('log_statsd_host') if statsd_host: statsd_port = int(conf.get('log_statsd_port', 8125)) base_prefix = conf.get('log_statsd_metric_prefix', '') default_sample_rate = float(conf.get( 'log_statsd_default_sample_rate', 1)) sample_rate_factor = float(conf.get( 'log_statsd_sample_rate_factor', 1)) statsd_client = StatsdClient(statsd_host, statsd_port, base_prefix, name, default_sample_rate, sample_rate_factor) logger.statsd_client = statsd_client else: logger.statsd_client = None adapted_logger = LogAdapter(logger, name) other_handlers = conf.get('log_custom_handlers', None) if other_handlers: log_custom_handlers = [s.strip() for s in other_handlers.split(',') if s.strip()] for hook in log_custom_handlers: try: mod, fnc = hook.rsplit('.', 1) logger_hook = getattr(__import__(mod, fromlist=[fnc]), fnc) logger_hook(conf, name, log_to_console, log_route, fmt, logger, adapted_logger) except (AttributeError, ImportError): print >>sys.stderr, 'Error calling custom handler [%s]' % hook except ValueError: print >>sys.stderr, 'Invalid custom handler format [%s]' % hook # Python 2.6 has the undesirable property of keeping references to all log # handlers around forever in logging._handlers and logging._handlerList. # Combine that with handlers that keep file descriptors, and you get an fd # leak. # # And no, we can't share handlers; a SyslogHandler has a socket, and if # two greenthreads end up logging at the same time, you could get message # overlap that garbles the logs and makes eventlet complain. # # Python 2.7 uses weakrefs to avoid the leak, so let's do that too. if sys.version_info[0] == 2 and sys.version_info[1] <= 6: try: logging._acquireLock() # some thread-safety thing for handler in adapted_logger.logger.handlers: if handler in logging._handlers: wr = LoggingHandlerWeakRef(handler) del logging._handlers[handler] logging._handlers[wr] = 1 for i, handler_ref in enumerate(logging._handlerList): if handler_ref is handler: logging._handlerList[i] = LoggingHandlerWeakRef( handler) finally: logging._releaseLock() return adapted_logger def get_hub(): """ Checks whether poll is available and falls back on select if it isn't. Note about epoll: Review: https://review.openstack.org/#/c/18806/ There was a problem where once out of every 30 quadrillion connections, a coroutine wouldn't wake up when the client closed its end. Epoll was not reporting the event or it was getting swallowed somewhere. Then when that file descriptor was re-used, eventlet would freak right out because it still thought it was waiting for activity from it in some other coro. """ try: import select if hasattr(select, "poll"): return "poll" return "selects" except ImportError: return None def drop_privileges(user): """ Sets the userid/groupid of the current process, get session leader, etc. :param user: User name to change privileges to """ if os.geteuid() == 0: groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem] os.setgroups(groups) user = pwd.getpwnam(user) os.setgid(user[3]) os.setuid(user[2]) os.environ['HOME'] = user[5] try: os.setsid() except OSError: pass os.chdir('/') # in case you need to rmdir on where you started the daemon os.umask(0o22) # ensure files are created with the correct privileges def capture_stdio(logger, **kwargs): """ Log unhandled exceptions, close stdio, capture stdout and stderr. param logger: Logger object to use """ # log uncaught exceptions sys.excepthook = lambda * exc_info: \ logger.critical(_('UNCAUGHT EXCEPTION'), exc_info=exc_info) # collect stdio file desc not in use for logging stdio_files = [sys.stdin, sys.stdout, sys.stderr] console_fds = [h.stream.fileno() for _junk, h in getattr( get_logger, 'console_handler4logger', {}).items()] stdio_files = [f for f in stdio_files if f.fileno() not in console_fds] with open(os.devnull, 'r+b') as nullfile: # close stdio (excludes fds open for logging) for f in stdio_files: # some platforms throw an error when attempting an stdin flush try: f.flush() except IOError: pass try: os.dup2(nullfile.fileno(), f.fileno()) except OSError: pass # redirect stdio if kwargs.pop('capture_stdout', True): sys.stdout = LoggerFileObject(logger) if kwargs.pop('capture_stderr', True): sys.stderr = LoggerFileObject(logger) def parse_options(parser=None, once=False, test_args=None): """ Parse standard swift server/daemon options with optparse.OptionParser. :param parser: OptionParser to use. If not sent one will be created. :param once: Boolean indicating the "once" option is available :param test_args: Override sys.argv; used in testing :returns : Tuple of (config, options); config is an absolute path to the config file, options is the parser options as a dictionary. :raises SystemExit: First arg (CONFIG) is required, file must exist """ if not parser: parser = OptionParser(usage="%prog CONFIG [options]") parser.add_option("-v", "--verbose", default=False, action="store_true", help="log to console") if once: parser.add_option("-o", "--once", default=False, action="store_true", help="only run one pass of daemon") # if test_args is None, optparse will use sys.argv[:1] options, args = parser.parse_args(args=test_args) if not args: parser.print_usage() print _("Error: missing config path argument") sys.exit(1) config = os.path.abspath(args.pop(0)) if not os.path.exists(config): parser.print_usage() print _("Error: unable to locate %s") % config sys.exit(1) extra_args = [] # if any named options appear in remaining args, set the option to True for arg in args: if arg in options.__dict__: setattr(options, arg, True) else: extra_args.append(arg) options = vars(options) if extra_args: options['extra_args'] = extra_args return config, options def whataremyips(): """ Get the machine's ip addresses :returns: list of Strings of ip addresses """ addresses = [] for interface in netifaces.interfaces(): try: iface_data = netifaces.ifaddresses(interface) for family in iface_data: if family not in (netifaces.AF_INET, netifaces.AF_INET6): continue for address in iface_data[family]: addr = address['addr'] # If we have an ipv6 address remove the # %ether_interface at the end if family == netifaces.AF_INET6: addr = addr.split('%')[0] addresses.append(addr) except ValueError: pass return addresses def storage_directory(datadir, partition, name_hash): """ Get the storage directory :param datadir: Base data directory :param partition: Partition :param name_hash: Account, container or object name hash :returns: Storage directory """ return os.path.join(datadir, str(partition), name_hash[-3:], name_hash) def hash_path(account, container=None, object=None, raw_digest=False): """ Get the canonical hash for an account/container/object :param account: Account :param container: Container :param object: Object :param raw_digest: If True, return the raw version rather than a hex digest :returns: hash string """ if object and not container: raise ValueError('container is required if object is provided') paths = [account] if container: paths.append(container) if object: paths.append(object) if raw_digest: return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + HASH_PATH_SUFFIX).digest() else: return md5(HASH_PATH_PREFIX + '/' + '/'.join(paths) + HASH_PATH_SUFFIX).hexdigest() @contextmanager def lock_path(directory, timeout=10, timeout_class=LockTimeout): """ Context manager that acquires a lock on a directory. This will block until the lock can be acquired, or the timeout time has expired (whichever occurs first). For locking exclusively, file or directory has to be opened in Write mode. Python doesn't allow directories to be opened in Write Mode. So we workaround by locking a hidden file in the directory. :param directory: directory to be locked :param timeout: timeout (in seconds) :param timeout_class: The class of the exception to raise if the lock cannot be granted within the timeout. Will be constructed as timeout_class(timeout, lockpath). Default: LockTimeout """ mkdirs(directory) lockpath = '%s/.lock' % directory fd = os.open(lockpath, os.O_WRONLY | os.O_CREAT) try: with timeout_class(timeout, lockpath): while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) break except IOError as err: if err.errno != errno.EAGAIN: raise sleep(0.01) yield True finally: os.close(fd) @contextmanager def lock_file(filename, timeout=10, append=False, unlink=True): """ Context manager that acquires a lock on a file. This will block until the lock can be acquired, or the timeout time has expired (whichever occurs first). :param filename: file to be locked :param timeout: timeout (in seconds) :param append: True if file should be opened in append mode :param unlink: True if the file should be unlinked at the end """ flags = os.O_CREAT | os.O_RDWR if append: flags |= os.O_APPEND mode = 'a+' else: mode = 'r+' fd = os.open(filename, flags) file_obj = os.fdopen(fd, mode) try: with LockTimeout(timeout, filename): while True: try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) break except IOError as err: if err.errno != errno.EAGAIN: raise sleep(0.01) yield file_obj finally: try: file_obj.close() except UnboundLocalError: pass # may have not actually opened the file if unlink: os.unlink(filename) def lock_parent_directory(filename, timeout=10): """ Context manager that acquires a lock on the parent directory of the given file path. This will block until the lock can be acquired, or the timeout time has expired (whichever occurs first). :param filename: file path of the parent directory to be locked :param timeout: timeout (in seconds) """ return lock_path(os.path.dirname(filename), timeout=timeout) def get_time_units(time_amount): """ Get a nomralized length of time in the largest unit of time (hours, minutes, or seconds.) :param time_amount: length of time in seconds :returns: A touple of (length of time, unit of time) where unit of time is one of ('h', 'm', 's') """ time_unit = 's' if time_amount > 60: time_amount /= 60 time_unit = 'm' if time_amount > 60: time_amount /= 60 time_unit = 'h' return time_amount, time_unit def compute_eta(start_time, current_value, final_value): """ Compute an ETA. Now only if we could also have a progress bar... :param start_time: Unix timestamp when the operation began :param current_value: Current value :param final_value: Final value :returns: ETA as a tuple of (length of time, unit of time) where unit of time is one of ('h', 'm', 's') """ elapsed = time.time() - start_time completion = (float(current_value) / final_value) or 0.00001 return get_time_units(1.0 / completion * elapsed - elapsed) def unlink_older_than(path, mtime): """ Remove any file in a given path that that was last modified before mtime. :param path: path to remove file from :mtime: timestamp of oldest file to keep """ for fname in listdir(path): fpath = os.path.join(path, fname) try: if os.path.getmtime(fpath) < mtime: os.unlink(fpath) except OSError: pass def item_from_env(env, item_name): """ Get a value from the wsgi environment :param env: wsgi environment dict :param item_name: name of item to get :returns: the value from the environment """ item = env.get(item_name, None) if item is None: logging.error("ERROR: %s could not be found in env!" % item_name) return item def cache_from_env(env): """ Get memcache connection pool from the environment (which had been previously set by the memcache middleware :param env: wsgi environment dict :returns: swift.common.memcached.MemcacheRing from environment """ return item_from_env(env, 'swift.cache') def read_conf_dir(parser, conf_dir): conf_files = [] for f in os.listdir(conf_dir): if f.endswith('.conf') and not f.startswith('.'): conf_files.append(os.path.join(conf_dir, f)) return parser.read(sorted(conf_files)) def readconf(conf_path, section_name=None, log_name=None, defaults=None, raw=False): """ Read config file(s) and return config items as a dict :param conf_path: path to config file/directory, or a file-like object (hasattr readline) :param section_name: config section to read (will return all sections if not defined) :param log_name: name to be used with logging (will use section_name if not defined) :param defaults: dict of default values to pre-populate the config with :returns: dict of config items """ if defaults is None: defaults = {} if raw: c = RawConfigParser(defaults) else: c = ConfigParser(defaults) if hasattr(conf_path, 'readline'): c.readfp(conf_path) else: if os.path.isdir(conf_path): # read all configs in directory success = read_conf_dir(c, conf_path) else: success = c.read(conf_path) if not success: print _("Unable to read config from %s") % conf_path sys.exit(1) if section_name: if c.has_section(section_name): conf = dict(c.items(section_name)) else: print _("Unable to find %s config section in %s") % \ (section_name, conf_path) sys.exit(1) if "log_name" not in conf: if log_name is not None: conf['log_name'] = log_name else: conf['log_name'] = section_name else: conf = {} for s in c.sections(): conf.update({s: dict(c.items(s))}) if 'log_name' not in conf: conf['log_name'] = log_name conf['__file__'] = conf_path return conf def write_pickle(obj, dest, tmp=None, pickle_protocol=0): """ Ensure that a pickle file gets written to disk. The file is first written to a tmp location, ensure it is synced to disk, then perform a move to its final location :param obj: python object to be pickled :param dest: path of final destination file :param tmp: path to tmp to use, defaults to None :param pickle_protocol: protocol to pickle the obj with, defaults to 0 """ if tmp is None: tmp = os.path.dirname(dest) fd, tmppath = mkstemp(dir=tmp, suffix='.tmp') with os.fdopen(fd, 'wb') as fo: pickle.dump(obj, fo, pickle_protocol) fo.flush() os.fsync(fd) renamer(tmppath, dest) def search_tree(root, glob_match, ext='', dir_ext=None): """Look in root, for any files/dirs matching glob, recursively traversing any found directories looking for files ending with ext :param root: start of search path :param glob_match: glob to match in root, matching dirs are traversed with os.walk :param ext: only files that end in ext will be returned :param dir_ext: if present directories that end with dir_ext will not be traversed and instead will be returned as a matched path :returns: list of full paths to matching files, sorted """ found_files = [] for path in glob.glob(os.path.join(root, glob_match)): if os.path.isdir(path): for root, dirs, files in os.walk(path): if dir_ext and root.endswith(dir_ext): found_files.append(root) # the root is a config dir, descend no further break for file_ in files: if ext and not file_.endswith(ext): continue found_files.append(os.path.join(root, file_)) found_dir = False for dir_ in dirs: if dir_ext and dir_.endswith(dir_ext): found_dir = True found_files.append(os.path.join(root, dir_)) if found_dir: # do not descend further into matching directories break else: if ext and not path.endswith(ext): continue found_files.append(path) return sorted(found_files) def write_file(path, contents): """Write contents to file at path :param path: any path, subdirs will be created as needed :param contents: data to write to file, will be converted to string """ dirname, name = os.path.split(path) if not os.path.exists(dirname): try: os.makedirs(dirname) except OSError as err: if err.errno == errno.EACCES: sys.exit('Unable to create %s. Running as ' 'non-root?' % dirname) with open(path, 'w') as f: f.write('%s' % contents) def remove_file(path): """Quiet wrapper for os.unlink, OSErrors are suppressed :param path: first and only argument passed to os.unlink """ try: os.unlink(path) except OSError: pass def audit_location_generator(devices, datadir, suffix='', mount_check=True, logger=None): ''' Given a devices path and a data directory, yield (path, device, partition) for all files in that directory :param devices: parent directory of the devices to be audited :param datadir: a directory located under self.devices. This should be one of the DATADIR constants defined in the account, container, and object servers. :param suffix: path name suffix required for all names returned :param mount_check: Flag to check if a mount check should be performed on devices :param logger: a logger object ''' device_dir = listdir(devices) # randomize devices in case of process restart before sweep completed shuffle(device_dir) for device in device_dir: if mount_check and not ismount(os.path.join(devices, device)): if logger: logger.debug( _('Skipping %s as it is not mounted'), device) continue datadir_path = os.path.join(devices, device, datadir) partitions = listdir(datadir_path) for partition in partitions: part_path = os.path.join(datadir_path, partition) try: suffixes = listdir(part_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for asuffix in suffixes: suff_path = os.path.join(part_path, asuffix) try: hashes = listdir(suff_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for hsh in hashes: hash_path = os.path.join(suff_path, hsh) try: files = sorted(listdir(hash_path), reverse=True) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for fname in files: if suffix and not fname.endswith(suffix): continue path = os.path.join(hash_path, fname) yield path, device, partition def ratelimit_sleep(running_time, max_rate, incr_by=1, rate_buffer=5): ''' Will eventlet.sleep() for the appropriate time so that the max_rate is never exceeded. If max_rate is 0, will not ratelimit. The maximum recommended rate should not exceed (1000 * incr_by) a second as eventlet.sleep() does involve some overhead. Returns running_time that should be used for subsequent calls. :param running_time: the running time in milliseconds of the next allowable request. Best to start at zero. :param max_rate: The maximum rate per second allowed for the process. :param incr_by: How much to increment the counter. Useful if you want to ratelimit 1024 bytes/sec and have differing sizes of requests. Must be > 0 to engage rate-limiting behavior. :param rate_buffer: Number of seconds the rate counter can drop and be allowed to catch up (at a faster than listed rate). A larger number will result in larger spikes in rate but better average accuracy. Must be > 0 to engage rate-limiting behavior. ''' if max_rate <= 0 or incr_by <= 0: return running_time # 1,000 milliseconds = 1 second clock_accuracy = 1000.0 # Convert seconds to milliseconds now = time.time() * clock_accuracy # Calculate time per request in milliseconds time_per_request = clock_accuracy * (float(incr_by) / max_rate) # Convert rate_buffer to milliseconds and compare if now - running_time > rate_buffer * clock_accuracy: running_time = now elif running_time - now > time_per_request: # Convert diff back to a floating point number of seconds and sleep eventlet.sleep((running_time - now) / clock_accuracy) # Return the absolute time for the next interval in milliseconds; note # that time could have passed well beyond that point, but the next call # will catch that and skip the sleep. return running_time + time_per_request class ContextPool(GreenPool): "GreenPool subclassed to kill its coros when it gets gc'ed" def __enter__(self): return self def __exit__(self, type, value, traceback): for coro in list(self.coroutines_running): coro.kill() class GreenAsyncPileWaitallTimeout(Timeout): pass class GreenAsyncPile(object): """ Runs jobs in a pool of green threads, and the results can be retrieved by using this object as an iterator. This is very similar in principle to eventlet.GreenPile, except it returns results as they become available rather than in the order they were launched. Correlating results with jobs (if necessary) is left to the caller. """ def __init__(self, size): """ :param size: size pool of green threads to use """ self._pool = GreenPool(size) self._responses = eventlet.queue.LightQueue(size) self._inflight = 0 def _run_func(self, func, args, kwargs): try: self._responses.put(func(*args, **kwargs)) finally: self._inflight -= 1 def spawn(self, func, *args, **kwargs): """ Spawn a job in a green thread on the pile. """ self._inflight += 1 self._pool.spawn(self._run_func, func, args, kwargs) def waitall(self, timeout): """ Wait timeout seconds for any results to come in. :param timeout: seconds to wait for results :returns: list of results accrued in that time """ results = [] try: with GreenAsyncPileWaitallTimeout(timeout): while True: results.append(self.next()) except (GreenAsyncPileWaitallTimeout, StopIteration): pass return results def __iter__(self): return self def next(self): try: return self._responses.get_nowait() except Empty: if self._inflight == 0: raise StopIteration() else: return self._responses.get() class ModifiedParseResult(ParseResult): "Parse results class for urlparse." @property def hostname(self): netloc = self.netloc.split('@', 1)[-1] if netloc.startswith('['): return netloc[1:].split(']')[0] elif ':' in netloc: return netloc.rsplit(':')[0] return netloc @property def port(self): netloc = self.netloc.split('@', 1)[-1] if netloc.startswith('['): netloc = netloc.rsplit(']')[1] if ':' in netloc: return int(netloc.rsplit(':')[1]) return None def urlparse(url): """ urlparse augmentation. This is necessary because urlparse can't handle RFC 2732 URLs. :param url: URL to parse. """ return ModifiedParseResult(*stdlib_urlparse(url)) def validate_sync_to(value, allowed_sync_hosts, realms_conf): """ Validates an X-Container-Sync-To header value, returning the validated endpoint, realm, and realm_key, or an error string. :param value: The X-Container-Sync-To header value to validate. :param allowed_sync_hosts: A list of allowed hosts in endpoints, if realms_conf does not apply. :param realms_conf: A instance of swift.common.container_sync_realms.ContainerSyncRealms to validate against. :returns: A tuple of (error_string, validated_endpoint, realm, realm_key). The error_string will None if the rest of the values have been validated. The validated_endpoint will be the validated endpoint to sync to. The realm and realm_key will be set if validation was done through realms_conf. """ orig_value = value value = value.rstrip('/') if not value: return (None, None, None, None) if value.startswith('//'): if not realms_conf: return (None, None, None, None) data = value[2:].split('/') if len(data) != 4: return ( _('Invalid X-Container-Sync-To format %r') % orig_value, None, None, None) realm, cluster, account, container = data realm_key = realms_conf.key(realm) if not realm_key: return (_('No realm key for %r') % realm, None, None, None) endpoint = realms_conf.endpoint(realm, cluster) if not endpoint: return ( _('No cluster endpoint for %r %r') % (realm, cluster), None, None, None) return ( None, '%s/%s/%s' % (endpoint.rstrip('/'), account, container), realm.upper(), realm_key) p = urlparse(value) if p.scheme not in ('http', 'https'): return ( _('Invalid scheme %r in X-Container-Sync-To, must be "//", ' '"http", or "https".') % p.scheme, None, None, None) if not p.path: return (_('Path required in X-Container-Sync-To'), None, None, None) if p.params or p.query or p.fragment: return ( _('Params, queries, and fragments not allowed in ' 'X-Container-Sync-To'), None, None, None) if p.hostname not in allowed_sync_hosts: return ( _('Invalid host %r in X-Container-Sync-To') % p.hostname, None, None, None) return (None, value, None, None) def affinity_key_function(affinity_str): """Turns an affinity config value into a function suitable for passing to sort(). After doing so, the array will be sorted with respect to the given ordering. For example, if affinity_str is "r1=1, r2z7=2, r2z8=2", then the array will be sorted with all nodes from region 1 (r1=1) first, then all the nodes from region 2 zones 7 and 8 (r2z7=2 and r2z8=2), then everything else. Note that the order of the pieces of affinity_str is irrelevant; the priority values are what comes after the equals sign. If affinity_str is empty or all whitespace, then the resulting function will not alter the ordering of the nodes. :param affinity_str: affinity config value, e.g. "r1z2=3" or "r1=1, r2z1=2, r2z2=2" :returns: single-argument function :raises: ValueError if argument invalid """ affinity_str = affinity_str.strip() if not affinity_str: return lambda x: 0 priority_matchers = [] pieces = [s.strip() for s in affinity_str.split(',')] for piece in pieces: # matches r= or rz= match = re.match("r(\d+)(?:z(\d+))?=(\d+)$", piece) if match: region, zone, priority = match.groups() region = int(region) priority = int(priority) zone = int(zone) if zone else None matcher = {'region': region, 'priority': priority} if zone is not None: matcher['zone'] = zone priority_matchers.append(matcher) else: raise ValueError("Invalid affinity value: %r" % affinity_str) priority_matchers.sort(key=operator.itemgetter('priority')) def keyfn(ring_node): for matcher in priority_matchers: if (matcher['region'] == ring_node['region'] and ('zone' not in matcher or matcher['zone'] == ring_node['zone'])): return matcher['priority'] return 4294967296 # 2^32, i.e. "a big number" return keyfn def affinity_locality_predicate(write_affinity_str): """ Turns a write-affinity config value into a predicate function for nodes. The returned value will be a 1-arg function that takes a node dictionary and returns a true value if it is "local" and a false value otherwise. The definition of "local" comes from the affinity_str argument passed in here. For example, if affinity_str is "r1, r2z2", then only nodes where region=1 or where (region=2 and zone=2) are considered local. If affinity_str is empty or all whitespace, then the resulting function will consider everything local :param affinity_str: affinity config value, e.g. "r1z2" or "r1, r2z1, r2z2" :returns: single-argument function, or None if affinity_str is empty :raises: ValueError if argument invalid """ affinity_str = write_affinity_str.strip() if not affinity_str: return None matchers = [] pieces = [s.strip() for s in affinity_str.split(',')] for piece in pieces: # matches r or rz match = re.match("r(\d+)(?:z(\d+))?$", piece) if match: region, zone = match.groups() region = int(region) zone = int(zone) if zone else None matcher = {'region': region} if zone is not None: matcher['zone'] = zone matchers.append(matcher) else: raise ValueError("Invalid write-affinity value: %r" % affinity_str) def is_local(ring_node): for matcher in matchers: if (matcher['region'] == ring_node['region'] and ('zone' not in matcher or matcher['zone'] == ring_node['zone'])): return True return False return is_local def get_remote_client(req): # remote host for zeus client = req.headers.get('x-cluster-client-ip') if not client and 'x-forwarded-for' in req.headers: # remote host for other lbs client = req.headers['x-forwarded-for'].split(',')[0].strip() if not client: client = req.remote_addr return client def human_readable(value): """ Returns the number in a human readable format; for example 1048576 = "1Mi". """ value = float(value) index = -1 suffixes = 'KMGTPEZY' while value >= 1024 and index + 1 < len(suffixes): index += 1 value = round(value / 1024) if index == -1: return '%d' % value return '%d%si' % (round(value), suffixes[index]) def put_recon_cache_entry(cache_entry, key, item): """ Function that will check if item is a dict, and if so put it under cache_entry[key]. We use nested recon cache entries when the object auditor runs in 'once' mode with a specified subset of devices. """ if isinstance(item, dict): if key not in cache_entry or key in cache_entry and not \ isinstance(cache_entry[key], dict): cache_entry[key] = {} elif key in cache_entry and item == {}: cache_entry.pop(key, None) return for k, v in item.items(): if v == {}: cache_entry[key].pop(k, None) else: cache_entry[key][k] = v else: cache_entry[key] = item def dump_recon_cache(cache_dict, cache_file, logger, lock_timeout=2): """Update recon cache values :param cache_dict: Dictionary of cache key/value pairs to write out :param cache_file: cache file to update :param logger: the logger to use to log an encountered error :param lock_timeout: timeout (in seconds) """ try: with lock_file(cache_file, lock_timeout, unlink=False) as cf: cache_entry = {} try: existing_entry = cf.readline() if existing_entry: cache_entry = json.loads(existing_entry) except ValueError: #file doesn't have a valid entry, we'll recreate it pass for cache_key, cache_value in cache_dict.items(): put_recon_cache_entry(cache_entry, cache_key, cache_value) try: with NamedTemporaryFile(dir=os.path.dirname(cache_file), delete=False) as tf: tf.write(json.dumps(cache_entry) + '\n') os.rename(tf.name, cache_file) finally: try: os.unlink(tf.name) except OSError as err: if err.errno != errno.ENOENT: raise except (Exception, Timeout): logger.exception(_('Exception dumping recon cache')) def listdir(path): try: return os.listdir(path) except OSError as err: if err.errno != errno.ENOENT: raise return [] def streq_const_time(s1, s2): """Constant-time string comparison. :params s1: the first string :params s2: the second string :return: True if the strings are equal. This function takes two strings and compares them. It is intended to be used when doing a comparison for authentication purposes to help guard against timing attacks. """ if len(s1) != len(s2): return False result = 0 for (a, b) in zip(s1, s2): result |= ord(a) ^ ord(b) return result == 0 def replication(func): """ Decorator to declare which methods are accessible for different type of servers: * If option replication_server is None then this decorator doesn't matter. * If option replication_server is True then ONLY decorated with this decorator methods will be started. * If option replication_server is False then decorated with this decorator methods will NOT be started. :param func: function to mark accessible for replication """ func.replication = True return func def public(func): """ Decorator to declare which methods are publicly accessible as HTTP requests :param func: function to make public """ func.publicly_accessible = True @functools.wraps(func) def wrapped(*a, **kw): return func(*a, **kw) return wrapped def quorum_size(n): """ Number of successful backend requests needed for the proxy to consider the client request successful. """ return (n // 2) + 1 def rsync_ip(ip): """ Transform ip string to an rsync-compatible form Will return ipv4 addresses unchanged, but will nest ipv6 addresses inside square brackets. :param ip: an ip string (ipv4 or ipv6) :returns: a string ip address """ try: socket.inet_pton(socket.AF_INET6, ip) except socket.error: # it's IPv4 return ip else: return '[%s]' % ip def get_valid_utf8_str(str_or_unicode): """ Get valid parts of utf-8 str from str, unicode and even invalid utf-8 str :param str_or_unicode: a string or an unicode which can be invalid utf-8 """ if isinstance(str_or_unicode, unicode): (str_or_unicode, _len) = utf8_encoder(str_or_unicode, 'replace') (valid_utf8_str, _len) = utf8_decoder(str_or_unicode, 'replace') return valid_utf8_str.encode('utf-8') def list_from_csv(comma_separated_str): """ Splits the str given and returns a properly stripped list of the comma separated values. """ if comma_separated_str: return [v.strip() for v in comma_separated_str.split(',') if v.strip()] return [] def csv_append(csv_string, item): """ Appends an item to a comma-separated string. If the comma-separated string is empty/None, just returns item. """ if csv_string: return ",".join((csv_string, item)) else: return item class CloseableChain(object): """ Like itertools.chain, but with a close method that will attempt to invoke its sub-iterators' close methods, if any. """ def __init__(self, *iterables): self.iterables = iterables def __iter__(self): return iter(itertools.chain(*(self.iterables))) def close(self): for it in self.iterables: close_method = getattr(it, 'close', None) if close_method: close_method() def reiterate(iterable): """ Consume the first item from an iterator, then re-chain it to the rest of the iterator. This is useful when you want to make sure the prologue to downstream generators have been executed before continuing. :param iterable: an iterable object """ if isinstance(iterable, (list, tuple)): return iterable else: iterator = iter(iterable) try: chunk = '' while not chunk: chunk = next(iterator) return CloseableChain([chunk], iterator) except StopIteration: return [] class InputProxy(object): """ File-like object that counts bytes read. To be swapped in for wsgi.input for accounting purposes. """ def __init__(self, wsgi_input): """ :param wsgi_input: file-like object to wrap the functionality of """ self.wsgi_input = wsgi_input self.bytes_received = 0 self.client_disconnect = False def read(self, *args, **kwargs): """ Pass read request to the underlying file-like object and add bytes read to total. """ try: chunk = self.wsgi_input.read(*args, **kwargs) except Exception: self.client_disconnect = True raise self.bytes_received += len(chunk) return chunk def readline(self, *args, **kwargs): """ Pass readline request to the underlying file-like object and add bytes read to total. """ try: line = self.wsgi_input.readline(*args, **kwargs) except Exception: self.client_disconnect = True raise self.bytes_received += len(line) return line def tpool_reraise(func, *args, **kwargs): """ Hack to work around Eventlet's tpool not catching and reraising Timeouts. """ def inner(): try: return func(*args, **kwargs) except BaseException as err: return err resp = tpool.execute(inner) if isinstance(resp, BaseException): raise resp return resp class ThreadPool(object): BYTE = 'a'.encode('utf-8') """ Perform blocking operations in background threads. Call its methods from within greenlets to green-wait for results without blocking the eventlet reactor (hopefully). """ def __init__(self, nthreads=2): self.nthreads = nthreads self._run_queue = Queue() self._result_queue = Queue() self._threads = [] if nthreads <= 0: return # We spawn a greenthread whose job it is to pull results from the # worker threads via a real Queue and send them to eventlet Events so # that the calling greenthreads can be awoken. # # Since each OS thread has its own collection of greenthreads, it # doesn't work to have the worker thread send stuff to the event, as # it then notifies its own thread-local eventlet hub to wake up, which # doesn't do anything to help out the actual calling greenthread over # in the main thread. # # Thus, each worker sticks its results into a result queue and then # writes a byte to a pipe, signaling the result-consuming greenlet (in # the main thread) to wake up and consume results. # # This is all stuff that eventlet.tpool does, but that code can't have # multiple instances instantiated. Since the object server uses one # pool per disk, we have to reimplement this stuff. _raw_rpipe, self.wpipe = os.pipe() self.rpipe = greenio.GreenPipe(_raw_rpipe, 'rb', bufsize=0) for _junk in xrange(nthreads): thr = stdlib_threading.Thread( target=self._worker, args=(self._run_queue, self._result_queue)) thr.daemon = True thr.start() self._threads.append(thr) # This is the result-consuming greenthread that runs in the main OS # thread, as described above. self._consumer_coro = greenthread.spawn_n(self._consume_results, self._result_queue) def _worker(self, work_queue, result_queue): """ Pulls an item from the queue and runs it, then puts the result into the result queue. Repeats forever. :param work_queue: queue from which to pull work :param result_queue: queue into which to place results """ while True: item = work_queue.get() ev, func, args, kwargs = item try: result = func(*args, **kwargs) result_queue.put((ev, True, result)) except BaseException: result_queue.put((ev, False, sys.exc_info())) finally: work_queue.task_done() os.write(self.wpipe, self.BYTE) def _consume_results(self, queue): """ Runs as a greenthread in the same OS thread as callers of run_in_thread(). Takes results from the worker OS threads and sends them to the waiting greenthreads. """ while True: try: self.rpipe.read(1) except ValueError: # can happen at process shutdown when pipe is closed break while True: try: ev, success, result = queue.get(block=False) except Empty: break try: if success: ev.send(result) else: ev.send_exception(*result) finally: queue.task_done() def run_in_thread(self, func, *args, **kwargs): """ Runs func(*args, **kwargs) in a thread. Blocks the current greenlet until results are available. Exceptions thrown will be reraised in the calling thread. If the threadpool was initialized with nthreads=0, it invokes func(*args, **kwargs) directly, followed by eventlet.sleep() to ensure the eventlet hub has a chance to execute. It is more likely the hub will be invoked when queuing operations to an external thread. :returns: result of calling func :raises: whatever func raises """ if self.nthreads <= 0: result = func(*args, **kwargs) sleep() return result ev = event.Event() self._run_queue.put((ev, func, args, kwargs), block=False) # blocks this greenlet (and only *this* greenlet) until the real # thread calls ev.send(). result = ev.wait() return result def _run_in_eventlet_tpool(self, func, *args, **kwargs): """ Really run something in an external thread, even if we haven't got any threads of our own. """ def inner(): try: return (True, func(*args, **kwargs)) except (Timeout, BaseException) as err: return (False, err) success, result = tpool.execute(inner) if success: return result else: raise result def force_run_in_thread(self, func, *args, **kwargs): """ Runs func(*args, **kwargs) in a thread. Blocks the current greenlet until results are available. Exceptions thrown will be reraised in the calling thread. If the threadpool was initialized with nthreads=0, uses eventlet.tpool to run the function. This is in contrast to run_in_thread(), which will (in that case) simply execute func in the calling thread. :returns: result of calling func :raises: whatever func raises """ if self.nthreads <= 0: return self._run_in_eventlet_tpool(func, *args, **kwargs) else: return self.run_in_thread(func, *args, **kwargs) def ismount(path): """ Test whether a path is a mount point. This will catch any exceptions and translate them into a False return value Use ismount_raw to have the exceptions raised instead. """ try: return ismount_raw(path) except OSError: return False def ismount_raw(path): """ Test whether a path is a mount point. Whereas ismount will catch any exceptions and just return False, this raw version will not catch exceptions. This is code hijacked from C Python 2.6.8, adapted to remove the extra lstat() system call. """ try: s1 = os.lstat(path) except os.error as err: if err.errno == errno.ENOENT: # It doesn't exist -- so not a mount point :-) return False raise if stat.S_ISLNK(s1.st_mode): # A symlink can never be a mount point return False s2 = os.lstat(os.path.join(path, '..')) dev1 = s1.st_dev dev2 = s2.st_dev if dev1 != dev2: # path/.. on a different device as path return True ino1 = s1.st_ino ino2 = s2.st_ino if ino1 == ino2: # path/.. is the same i-node as path return True return False _rfc_token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+' _rfc_extension_pattern = re.compile( r'(?:\s*;\s*(' + _rfc_token + r")\s*(?:=\s*(" + _rfc_token + r'|"(?:[^"\\]|\\.)*"))?)') def parse_content_type(content_type): """ Parse a content-type and its parameters into values. RFC 2616 sec 14.17 and 3.7 are pertinent. **Examples**:: 'text/plain; charset=UTF-8' -> ('text/plain', [('charset, 'UTF-8')]) 'text/plain; charset=UTF-8; level=1' -> ('text/plain', [('charset, 'UTF-8'), ('level', '1')]) :param content_type: content_type to parse :returns: a typle containing (content type, list of k, v parameter tuples) """ parm_list = [] if ';' in content_type: content_type, parms = content_type.split(';', 1) parms = ';' + parms for m in _rfc_extension_pattern.findall(parms): key = m[0].strip() value = m[1].strip() parm_list.append((key, value)) return content_type, parm_list def override_bytes_from_content_type(listing_dict, logger=None): """ Takes a dict from a container listing and overrides the content_type, bytes fields if swift_bytes is set. """ content_type, params = parse_content_type(listing_dict['content_type']) for key, value in params: if key == 'swift_bytes': try: listing_dict['bytes'] = int(value) except ValueError: if logger: logger.exception("Invalid swift_bytes") else: content_type += ';%s=%s' % (key, value) listing_dict['content_type'] = content_type def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf-8 strings before quoting """ return _quote(get_valid_utf8_str(value), safe) swift-1.13.1/swift/common/container_sync_realms.py0000664000175400017540000001362312323703611023451 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import ConfigParser import errno import hashlib import hmac import os import time from swift import gettext_ as _ from swift.common.utils import get_valid_utf8_str class ContainerSyncRealms(object): """ Loads and parses the container-sync-realms.conf, occasionally checking the file's mtime to see if it needs to be reloaded. """ def __init__(self, conf_path, logger): self.conf_path = conf_path self.logger = logger self.next_mtime_check = 0 self.mtime_check_interval = 300 self.conf_path_mtime = 0 self.data = {} self.reload() def reload(self): """Forces a reload of the conf file.""" self.next_mtime_check = 0 self.conf_path_mtime = 0 self._reload() def _reload(self): now = time.time() if now >= self.next_mtime_check: self.next_mtime_check = now + self.mtime_check_interval try: mtime = os.path.getmtime(self.conf_path) except OSError as err: if err.errno == errno.ENOENT: log_func = self.logger.debug else: log_func = self.logger.error log_func(_('Could not load %r: %s'), self.conf_path, err) else: if mtime != self.conf_path_mtime: self.conf_path_mtime = mtime try: conf = ConfigParser.SafeConfigParser() conf.read(self.conf_path) except ConfigParser.ParsingError as err: self.logger.error( _('Could not load %r: %s'), self.conf_path, err) else: try: self.mtime_check_interval = conf.getint( 'DEFAULT', 'mtime_check_interval') self.next_mtime_check = \ now + self.mtime_check_interval except ConfigParser.NoOptionError: self.mtime_check_interval = 300 self.next_mtime_check = \ now + self.mtime_check_interval except (ConfigParser.ParsingError, ValueError) as err: self.logger.error( _('Error in %r with mtime_check_interval: %s'), self.conf_path, err) realms = {} for section in conf.sections(): realm = {} clusters = {} for option, value in conf.items(section): if option in ('key', 'key2'): realm[option] = value elif option.startswith('cluster_'): clusters[option[8:].upper()] = value realm['clusters'] = clusters realms[section.upper()] = realm self.data = realms def realms(self): """Returns a list of realms.""" self._reload() return self.data.keys() def key(self, realm): """Returns the key for the realm.""" self._reload() result = self.data.get(realm.upper()) if result: result = result.get('key') return result def key2(self, realm): """Returns the key2 for the realm.""" self._reload() result = self.data.get(realm.upper()) if result: result = result.get('key2') return result def clusters(self, realm): """Returns a list of clusters for the realm.""" self._reload() result = self.data.get(realm.upper()) if result: result = result.get('clusters') if result: result = result.keys() return result or [] def endpoint(self, realm, cluster): """Returns the endpoint for the cluster in the realm.""" self._reload() result = None realm_data = self.data.get(realm.upper()) if realm_data: cluster_data = realm_data.get('clusters') if cluster_data: result = cluster_data.get(cluster.upper()) return result def get_sig(self, request_method, path, x_timestamp, nonce, realm_key, user_key): """ Returns the hexdigest string of the HMAC-SHA1 (RFC 2104) for the information given. :param request_method: HTTP method of the request. :param path: The path to the resource. :param x_timestamp: The X-Timestamp header value for the request. :param nonce: A unique value for the request. :param realm_key: Shared secret at the cluster operator level. :param user_key: Shared secret at the user's container level. :returns: hexdigest str of the HMAC-SHA1 for the request. """ nonce = get_valid_utf8_str(nonce) realm_key = get_valid_utf8_str(realm_key) user_key = get_valid_utf8_str(user_key) return hmac.new( realm_key, '%s\n%s\n%s\n%s\n%s' % ( request_method, path, x_timestamp, nonce, user_key), hashlib.sha1).hexdigest() swift-1.13.1/swift/common/request_helpers.py0000664000175400017540000003622612323703611022306 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Miscellaneous utility functions for use in generating responses. Why not swift.common.utils, you ask? Because this way we can import things from swob in here without creating circular imports. """ import hashlib import sys import time from contextlib import contextmanager from urllib import unquote from swift.common.constraints import FORMAT2CONTENT_TYPE from swift.common.exceptions import ListingIterError, SegmentError from swift.common.http import is_success, HTTP_SERVICE_UNAVAILABLE from swift.common.swob import HTTPBadRequest, HTTPNotAcceptable from swift.common.utils import split_path, validate_device_partition from swift.common.wsgi import make_subrequest def get_param(req, name, default=None): """ Get parameters from an HTTP request ensuring proper handling UTF-8 encoding. :param req: request object :param name: parameter name :param default: result to return if the parameter is not found :returns: HTTP request parameter value (as UTF-8 encoded str, not unicode object) :raises: HTTPBadRequest if param not valid UTF-8 byte sequence """ value = req.params.get(name, default) if value and not isinstance(value, unicode): try: value.decode('utf8') # Ensure UTF8ness except UnicodeDecodeError: raise HTTPBadRequest( request=req, content_type='text/plain', body='"%s" parameter not valid UTF-8' % name) return value def get_listing_content_type(req): """ Determine the content type to use for an account or container listing response. :param req: request object :returns: content type as a string (e.g. text/plain, application/json) :raises: HTTPNotAcceptable if the requested content type is not acceptable :raises: HTTPBadRequest if the 'format' query param is provided and not valid UTF-8 """ query_format = get_param(req, 'format') if query_format: req.accept = FORMAT2CONTENT_TYPE.get( query_format.lower(), FORMAT2CONTENT_TYPE['plain']) out_content_type = req.accept.best_match( ['text/plain', 'application/json', 'application/xml', 'text/xml']) if not out_content_type: raise HTTPNotAcceptable(request=req) return out_content_type def split_and_validate_path(request, minsegs=1, maxsegs=None, rest_with_last=False): """ Utility function to split and validate the request path. :returns: result of split_path if everything's okay :raises: HTTPBadRequest if something's not okay """ try: segs = split_path(unquote(request.path), minsegs, maxsegs, rest_with_last) validate_device_partition(segs[0], segs[1]) return segs except ValueError as err: raise HTTPBadRequest(body=str(err), request=request, content_type='text/plain') def is_user_meta(server_type, key): """ Tests if a header key starts with and is longer than the user metadata prefix for given server type. :param server_type: type of backend server i.e. [account|container|object] :param key: header key :returns: True if the key satisfies the test, False otherwise """ if len(key) <= 8 + len(server_type): return False return key.lower().startswith(get_user_meta_prefix(server_type)) def is_sys_meta(server_type, key): """ Tests if a header key starts with and is longer than the system metadata prefix for given server type. :param server_type: type of backend server i.e. [account|container|object] :param key: header key :returns: True if the key satisfies the test, False otherwise """ if len(key) <= 11 + len(server_type): return False return key.lower().startswith(get_sys_meta_prefix(server_type)) def is_sys_or_user_meta(server_type, key): """ Tests if a header key starts with and is longer than the user or system metadata prefix for given server type. :param server_type: type of backend server i.e. [account|container|object] :param key: header key :returns: True if the key satisfies the test, False otherwise """ return is_user_meta(server_type, key) or is_sys_meta(server_type, key) def strip_user_meta_prefix(server_type, key): """ Removes the user metadata prefix for a given server type from the start of a header key. :param server_type: type of backend server i.e. [account|container|object] :param key: header key :returns: stripped header key """ return key[len(get_user_meta_prefix(server_type)):] def strip_sys_meta_prefix(server_type, key): """ Removes the system metadata prefix for a given server type from the start of a header key. :param server_type: type of backend server i.e. [account|container|object] :param key: header key :returns: stripped header key """ return key[len(get_sys_meta_prefix(server_type)):] def get_user_meta_prefix(server_type): """ Returns the prefix for user metadata headers for given server type. This prefix defines the namespace for headers that will be persisted by backend servers. :param server_type: type of backend server i.e. [account|container|object] :returns: prefix string for server type's user metadata headers """ return 'x-%s-%s-' % (server_type.lower(), 'meta') def get_sys_meta_prefix(server_type): """ Returns the prefix for system metadata headers for given server type. This prefix defines the namespace for headers that will be persisted by backend servers. :param server_type: type of backend server i.e. [account|container|object] :returns: prefix string for server type's system metadata headers """ return 'x-%s-%s-' % (server_type.lower(), 'sysmeta') def remove_items(headers, condition): """ Removes items from a dict whose keys satisfy the given condition. :param headers: a dict of headers :param condition: a function that will be passed the header key as a single argument and should return True if the header is to be removed. :returns: a dict, possibly empty, of headers that have been removed """ removed = {} keys = filter(condition, headers) removed.update((key, headers.pop(key)) for key in keys) return removed def close_if_possible(maybe_closable): close_method = getattr(maybe_closable, 'close', None) if callable(close_method): return close_method() @contextmanager def closing_if_possible(maybe_closable): """ Like contextlib.closing(), but doesn't crash if the object lacks a close() method. PEP 333 (WSGI) says: "If the iterable returned by the application has a close() method, the server or gateway must call that method upon completion of the current request[.]" This function makes that easier. """ yield maybe_closable close_if_possible(maybe_closable) class SegmentedIterable(object): """ Iterable that returns the object contents for a large object. :param req: original request object :param app: WSGI application from which segments will come :param listing_iter: iterable yielding the object segments to fetch, along with the byte subranges to fetch, in the form of a tuple (object-path, first-byte, last-byte) or (object-path, None, None) to fetch the whole thing. :param max_get_time: maximum permitted duration of a GET request (seconds) :param logger: logger object :param swift_source: value of swift.source in subrequest environ (just for logging) :param ua_suffix: string to append to user-agent. :param name: name of manifest (used in logging only) :param response: optional response object for the response being sent to the client. """ def __init__(self, req, app, listing_iter, max_get_time, logger, ua_suffix, swift_source, name='', response=None): self.req = req self.app = app self.listing_iter = listing_iter self.max_get_time = max_get_time self.logger = logger self.ua_suffix = " " + ua_suffix self.swift_source = swift_source self.name = name self.response = response def app_iter_range(self, *a, **kw): """ swob.Response will only respond with a 206 status in certain cases; one of those is if the body iterator responds to .app_iter_range(). However, this object (or really, its listing iter) is smart enough to handle the range stuff internally, so we just no-op this out for swob. """ return self def __iter__(self): start_time = time.time() have_yielded_data = False if self.response and self.response.content_length: bytes_left = int(self.response.content_length) else: bytes_left = None try: for seg_path, seg_etag, seg_size, first_byte, last_byte \ in self.listing_iter: if time.time() - start_time > self.max_get_time: raise SegmentError( 'ERROR: While processing manifest %s, ' 'max LO GET time of %ds exceeded' % (self.name, self.max_get_time)) # Make sure that the segment is a plain old object, not some # flavor of large object, so that we can check its MD5. path = seg_path + '?multipart-manifest=get' seg_req = make_subrequest( self.req.environ, path=path, method='GET', headers={'x-auth-token': self.req.headers.get( 'x-auth-token')}, agent=('%(orig)s ' + self.ua_suffix), swift_source=self.swift_source) if first_byte is not None or last_byte is not None: seg_req.headers['Range'] = "bytes=%s-%s" % ( # The 0 is to avoid having a range like "bytes=-10", # which actually means the *last* 10 bytes. '0' if first_byte is None else first_byte, '' if last_byte is None else last_byte) seg_resp = seg_req.get_response(self.app) if not is_success(seg_resp.status_int): close_if_possible(seg_resp.app_iter) raise SegmentError( 'ERROR: While processing manifest %s, ' 'got %d while retrieving %s' % (self.name, seg_resp.status_int, seg_path)) elif ((seg_etag and (seg_resp.etag != seg_etag)) or (seg_size and (seg_resp.content_length != seg_size) and not seg_req.range)): # The content-length check is for security reasons. Seems # possible that an attacker could upload a >1mb object and # then replace it with a much smaller object with same # etag. Then create a big nested SLO that calls that # object many times which would hammer our obj servers. If # this is a range request, don't check content-length # because it won't match. close_if_possible(seg_resp.app_iter) raise SegmentError( 'Object segment no longer valid: ' '%(path)s etag: %(r_etag)s != %(s_etag)s or ' '%(r_size)s != %(s_size)s.' % {'path': seg_req.path, 'r_etag': seg_resp.etag, 'r_size': seg_resp.content_length, 's_etag': seg_etag, 's_size': seg_size}) seg_hash = hashlib.md5() for chunk in seg_resp.app_iter: seg_hash.update(chunk) have_yielded_data = True if bytes_left is None: yield chunk elif bytes_left >= len(chunk): yield chunk bytes_left -= len(chunk) else: yield chunk[:bytes_left] bytes_left -= len(chunk) close_if_possible(seg_resp.app_iter) raise SegmentError( 'Too many bytes for %(name)s; truncating in ' '%(seg)s with %(left)d bytes left' % {'name': self.name, 'seg': seg_req.path, 'left': bytes_left}) close_if_possible(seg_resp.app_iter) if seg_resp.etag and seg_hash.hexdigest() != seg_resp.etag \ and first_byte is None and last_byte is None: raise SegmentError( "Bad MD5 checksum in %(name)s for %(seg)s: headers had" " %(etag)s, but object MD5 was actually %(actual)s" % {'seg': seg_req.path, 'etag': seg_resp.etag, 'name': self.name, 'actual': seg_hash.hexdigest()}) if bytes_left: raise SegmentError( 'Not enough bytes for %s; closing connection' % self.name) except ListingIterError as err: # I have to save this error because yielding the ' ' below clears # the exception from the current stack frame. excinfo = sys.exc_info() self.logger.exception('ERROR: While processing manifest %s, %s', self.name, err) # Normally, exceptions before any data has been yielded will # cause Eventlet to send a 5xx response. In this particular # case of ListingIterError we don't want that and we'd rather # just send the normal 2xx response and then hang up early # since 5xx codes are often used to judge Service Level # Agreements and this ListingIterError indicates the user has # created an invalid condition. if not have_yielded_data: yield ' ' raise excinfo except SegmentError as err: self.logger.exception(err) # This doesn't actually change the response status (we're too # late for that), but this does make it to the logs. if self.response: self.response.status = HTTP_SERVICE_UNAVAILABLE raise swift-1.13.1/swift/common/exceptions.py0000664000175400017540000001020112323703611021236 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from eventlet import Timeout class MessageTimeout(Timeout): def __init__(self, seconds=None, msg=None): Timeout.__init__(self, seconds=seconds) self.msg = msg def __str__(self): return '%s: %s' % (Timeout.__str__(self), self.msg) class SwiftException(Exception): pass class DiskFileError(SwiftException): pass class DiskFileNotOpen(DiskFileError): pass class DiskFileQuarantined(DiskFileError): pass class DiskFileCollision(DiskFileError): pass class DiskFileNotExist(DiskFileError): pass class DiskFileDeleted(DiskFileNotExist): def __init__(self, metadata=None): self.metadata = metadata or {} self.timestamp = self.metadata.get('X-Timestamp', 0) class DiskFileExpired(DiskFileDeleted): pass class DiskFileNoSpace(DiskFileError): pass class DiskFileDeviceUnavailable(DiskFileError): pass class PathNotDir(OSError): pass class ChunkReadTimeout(Timeout): pass class ChunkWriteTimeout(Timeout): pass class ConnectionTimeout(Timeout): pass class DriveNotMounted(SwiftException): pass class LockTimeout(MessageTimeout): pass class RingBuilderError(SwiftException): pass class RingValidationError(RingBuilderError): pass class EmptyRingError(RingBuilderError): pass class DuplicateDeviceError(RingBuilderError): pass class ListingIterError(SwiftException): pass class ListingIterNotFound(ListingIterError): pass class ListingIterNotAuthorized(ListingIterError): def __init__(self, aresp): self.aresp = aresp class SegmentError(SwiftException): pass class ReplicationException(Exception): pass class ReplicationLockTimeout(LockTimeout): pass class ClientException(Exception): def __init__(self, msg, http_scheme='', http_host='', http_port='', http_path='', http_query='', http_status=0, http_reason='', http_device='', http_response_content=''): Exception.__init__(self, msg) self.msg = msg self.http_scheme = http_scheme self.http_host = http_host self.http_port = http_port self.http_path = http_path self.http_query = http_query self.http_status = http_status self.http_reason = http_reason self.http_device = http_device self.http_response_content = http_response_content def __str__(self): a = self.msg b = '' if self.http_scheme: b += '%s://' % self.http_scheme if self.http_host: b += self.http_host if self.http_port: b += ':%s' % self.http_port if self.http_path: b += self.http_path if self.http_query: b += '?%s' % self.http_query if self.http_status: if b: b = '%s %s' % (b, self.http_status) else: b = str(self.http_status) if self.http_reason: if b: b = '%s %s' % (b, self.http_reason) else: b = '- %s' % self.http_reason if self.http_device: if b: b = '%s: device %s' % (b, self.http_device) else: b = 'device %s' % self.http_device if self.http_response_content: if len(self.http_response_content) <= 60: b += ' %s' % self.http_response_content else: b += ' [first 60 chars of response] %s' \ % self.http_response_content[:60] return b and '%s: %s' % (a, b) or a swift-1.13.1/swift/common/db.py0000664000175400017540000007062112323703611017456 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Database code for Swift """ from contextlib import contextmanager, closing import hashlib import logging import os from uuid import uuid4 import sys import time import errno from swift import gettext_ as _ from tempfile import mkstemp from eventlet import sleep, Timeout import sqlite3 from swift.common.utils import json, normalize_timestamp, renamer, \ mkdirs, lock_parent_directory, fallocate from swift.common.exceptions import LockTimeout #: Whether calls will be made to preallocate disk space for database files. DB_PREALLOCATION = True #: Timeout for trying to connect to a DB BROKER_TIMEOUT = 25 #: Pickle protocol to use PICKLE_PROTOCOL = 2 #: Max number of pending entries PENDING_CAP = 131072 def utf8encode(*args): return [(s.encode('utf8') if isinstance(s, unicode) else s) for s in args] def utf8encodekeys(metadata): uni_keys = [k for k in metadata if isinstance(k, unicode)] for k in uni_keys: sv = metadata[k] del metadata[k] metadata[k.encode('utf-8')] = sv def _db_timeout(timeout, db_file, call): with LockTimeout(timeout, db_file): retry_wait = 0.001 while True: try: return call() except sqlite3.OperationalError as e: if 'locked' not in str(e): raise sleep(retry_wait) retry_wait = min(retry_wait * 2, 0.05) class DatabaseConnectionError(sqlite3.DatabaseError): """More friendly error messages for DB Errors.""" def __init__(self, path, msg, timeout=0): self.path = path self.timeout = timeout self.msg = msg def __str__(self): return 'DB connection error (%s, %s):\n%s' % ( self.path, self.timeout, self.msg) class DatabaseAlreadyExists(sqlite3.DatabaseError): """More friendly error messages for DB Errors.""" def __init__(self, path): self.path = path def __str__(self): return 'DB %s already exists' % self.path class GreenDBConnection(sqlite3.Connection): """SQLite DB Connection handler that plays well with eventlet.""" def __init__(self, database, timeout=None, *args, **kwargs): if timeout is None: timeout = BROKER_TIMEOUT self.timeout = timeout self.db_file = database super(GreenDBConnection, self).__init__(database, 0, *args, **kwargs) def cursor(self, cls=None): if cls is None: cls = GreenDBCursor return sqlite3.Connection.cursor(self, cls) def commit(self): return _db_timeout( self.timeout, self.db_file, lambda: sqlite3.Connection.commit(self)) class GreenDBCursor(sqlite3.Cursor): """SQLite Cursor handler that plays well with eventlet.""" def __init__(self, *args, **kwargs): self.timeout = args[0].timeout self.db_file = args[0].db_file super(GreenDBCursor, self).__init__(*args, **kwargs) def execute(self, *args, **kwargs): return _db_timeout( self.timeout, self.db_file, lambda: sqlite3.Cursor.execute( self, *args, **kwargs)) def dict_factory(crs, row): """ This should only be used when you need a real dict, i.e. when you're going to serialize the results. """ return dict( ((col[0], row[idx]) for idx, col in enumerate(crs.description))) def chexor(old, name, timestamp): """ Each entry in the account and container databases is XORed by the 128-bit hash on insert or delete. This serves as a rolling, order-independent hash of the contents. (check + XOR) :param old: hex representation of the current DB hash :param name: name of the object or container being inserted :param timestamp: timestamp of the new record :returns: a hex representation of the new hash value """ if name is None: raise Exception('name is None!') new = hashlib.md5(('%s-%s' % (name, timestamp)).encode('utf8')).hexdigest() return '%032x' % (int(old, 16) ^ int(new, 16)) def get_db_connection(path, timeout=30, okay_to_create=False): """ Returns a properly configured SQLite database connection. :param path: path to DB :param timeout: timeout for connection :param okay_to_create: if True, create the DB if it doesn't exist :returns: DB connection object """ try: connect_time = time.time() conn = sqlite3.connect(path, check_same_thread=False, factory=GreenDBConnection, timeout=timeout) if path != ':memory:' and not okay_to_create: # attempt to detect and fail when connect creates the db file stat = os.stat(path) if stat.st_size == 0 and stat.st_ctime >= connect_time: os.unlink(path) raise DatabaseConnectionError(path, 'DB file created by connect?') conn.row_factory = sqlite3.Row conn.text_factory = str with closing(conn.cursor()) as cur: cur.execute('PRAGMA synchronous = NORMAL') cur.execute('PRAGMA count_changes = OFF') cur.execute('PRAGMA temp_store = MEMORY') cur.execute('PRAGMA journal_mode = DELETE') conn.create_function('chexor', 3, chexor) except sqlite3.DatabaseError: import traceback raise DatabaseConnectionError(path, traceback.format_exc(), timeout=timeout) return conn class DatabaseBroker(object): """Encapsulates working with a database.""" def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, account=None, container=None, pending_timeout=None, stale_reads_ok=False): """Encapsulates working with a database.""" self.conn = None self.db_file = db_file self.pending_file = self.db_file + '.pending' self.pending_timeout = pending_timeout or 10 self.stale_reads_ok = stale_reads_ok self.db_dir = os.path.dirname(db_file) self.timeout = timeout self.logger = logger or logging.getLogger() self.account = account self.container = container self._db_version = -1 def __str__(self): """ Returns a string indentifying the entity under broker to a human. The baseline implementation returns a full pathname to a database. This is vital for useful diagnostics. """ return self.db_file def initialize(self, put_timestamp=None): """ Create the DB :param put_timestamp: timestamp of initial PUT request """ if self.db_file == ':memory:': tmp_db_file = None conn = get_db_connection(self.db_file, self.timeout) else: mkdirs(self.db_dir) fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir) os.close(fd) conn = sqlite3.connect(tmp_db_file, check_same_thread=False, factory=GreenDBConnection, timeout=0) # creating dbs implicitly does a lot of transactions, so we # pick fast, unsafe options here and do a big fsync at the end. with closing(conn.cursor()) as cur: cur.execute('PRAGMA synchronous = OFF') cur.execute('PRAGMA temp_store = MEMORY') cur.execute('PRAGMA journal_mode = MEMORY') conn.create_function('chexor', 3, chexor) conn.row_factory = sqlite3.Row conn.text_factory = str conn.executescript(""" CREATE TABLE outgoing_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 ); CREATE TABLE incoming_sync ( remote_id TEXT UNIQUE, sync_point INTEGER, updated_at TEXT DEFAULT 0 ); CREATE TRIGGER outgoing_sync_insert AFTER INSERT ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END; CREATE TRIGGER outgoing_sync_update AFTER UPDATE ON outgoing_sync BEGIN UPDATE outgoing_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END; CREATE TRIGGER incoming_sync_insert AFTER INSERT ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END; CREATE TRIGGER incoming_sync_update AFTER UPDATE ON incoming_sync BEGIN UPDATE incoming_sync SET updated_at = STRFTIME('%s', 'NOW') WHERE ROWID = new.ROWID; END; """) if not put_timestamp: put_timestamp = normalize_timestamp(0) self._initialize(conn, put_timestamp) conn.commit() if tmp_db_file: conn.close() with open(tmp_db_file, 'r+b') as fp: os.fsync(fp.fileno()) with lock_parent_directory(self.db_file, self.pending_timeout): if os.path.exists(self.db_file): # It's as if there was a "condition" where different parts # of the system were "racing" each other. raise DatabaseAlreadyExists(self.db_file) renamer(tmp_db_file, self.db_file) self.conn = get_db_connection(self.db_file, self.timeout) else: self.conn = conn def delete_db(self, timestamp): """ Mark the DB as deleted :param timestamp: delete timestamp """ timestamp = normalize_timestamp(timestamp) # first, clear the metadata cleared_meta = {} for k in self.metadata: cleared_meta[k] = ('', timestamp) self.update_metadata(cleared_meta) # then mark the db as deleted with self.get() as conn: self._delete_db(conn, timestamp) conn.commit() def possibly_quarantine(self, exc_type, exc_value, exc_traceback): """ Checks the exception info to see if it indicates a quarantine situation (malformed or corrupted database). If not, the original exception will be reraised. If so, the database will be quarantined and a new sqlite3.DatabaseError will be raised indicating the action taken. """ if 'database disk image is malformed' in str(exc_value): exc_hint = 'malformed' elif 'file is encrypted or is not a database' in str(exc_value): exc_hint = 'corrupted' else: raise exc_type, exc_value, exc_traceback prefix_path = os.path.dirname(self.db_dir) partition_path = os.path.dirname(prefix_path) dbs_path = os.path.dirname(partition_path) device_path = os.path.dirname(dbs_path) quar_path = os.path.join(device_path, 'quarantined', self.db_type + 's', os.path.basename(self.db_dir)) try: renamer(self.db_dir, quar_path) except OSError as e: if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): raise quar_path = "%s-%s" % (quar_path, uuid4().hex) renamer(self.db_dir, quar_path) detail = _('Quarantined %s to %s due to %s database') % \ (self.db_dir, quar_path, exc_hint) self.logger.error(detail) raise sqlite3.DatabaseError(detail) @contextmanager def get(self): """Use with the "with" statement; returns a database connection.""" if not self.conn: if self.db_file != ':memory:' and os.path.exists(self.db_file): try: self.conn = get_db_connection(self.db_file, self.timeout) except (sqlite3.DatabaseError, DatabaseConnectionError): self.possibly_quarantine(*sys.exc_info()) else: raise DatabaseConnectionError(self.db_file, "DB doesn't exist") conn = self.conn self.conn = None try: yield conn conn.rollback() self.conn = conn except sqlite3.DatabaseError: try: conn.close() except Exception: pass self.possibly_quarantine(*sys.exc_info()) except (Exception, Timeout): conn.close() raise @contextmanager def lock(self): """Use with the "with" statement; locks a database.""" if not self.conn: if self.db_file != ':memory:' and os.path.exists(self.db_file): self.conn = get_db_connection(self.db_file, self.timeout) else: raise DatabaseConnectionError(self.db_file, "DB doesn't exist") conn = self.conn self.conn = None orig_isolation_level = conn.isolation_level conn.isolation_level = None conn.execute('BEGIN IMMEDIATE') try: yield True except (Exception, Timeout): pass try: conn.execute('ROLLBACK') conn.isolation_level = orig_isolation_level self.conn = conn except (Exception, Timeout): logging.exception( _('Broker error trying to rollback locked connection')) conn.close() def newid(self, remote_id): """ Re-id the database. This should be called after an rsync. :param remote_id: the ID of the remote database being rsynced in """ with self.get() as conn: row = conn.execute(''' UPDATE %s_stat SET id=? ''' % self.db_type, (str(uuid4()),)) row = conn.execute(''' SELECT ROWID FROM %s ORDER BY ROWID DESC LIMIT 1 ''' % self.db_contains_type).fetchone() sync_point = row['ROWID'] if row else -1 conn.execute(''' INSERT OR REPLACE INTO incoming_sync (sync_point, remote_id) VALUES (?, ?) ''', (sync_point, remote_id)) self._newid(conn) conn.commit() def _newid(self, conn): # Override for additional work when receiving an rsynced db. pass def merge_timestamps(self, created_at, put_timestamp, delete_timestamp): """ Used in replication to handle updating timestamps. :param created_at: create timestamp :param put_timestamp: put timestamp :param delete_timestamp: delete timestamp """ with self.get() as conn: conn.execute(''' UPDATE %s_stat SET created_at=MIN(?, created_at), put_timestamp=MAX(?, put_timestamp), delete_timestamp=MAX(?, delete_timestamp) ''' % self.db_type, (created_at, put_timestamp, delete_timestamp)) conn.commit() def get_items_since(self, start, count): """ Get a list of objects in the database between start and end. :param start: start ROWID :param count: number to get :returns: list of objects between start and end """ self._commit_puts_stale_ok() with self.get() as conn: curs = conn.execute(''' SELECT * FROM %s WHERE ROWID > ? ORDER BY ROWID ASC LIMIT ? ''' % self.db_contains_type, (start, count)) curs.row_factory = dict_factory return [r for r in curs] def get_sync(self, id, incoming=True): """ Gets the most recent sync point for a server from the sync table. :param id: remote ID to get the sync_point for :param incoming: if True, get the last incoming sync, otherwise get the last outgoing sync :returns: the sync point, or -1 if the id doesn't exist. """ with self.get() as conn: row = conn.execute( "SELECT sync_point FROM %s_sync WHERE remote_id=?" % ('incoming' if incoming else 'outgoing'), (id,)).fetchone() if not row: return -1 return row['sync_point'] def get_syncs(self, incoming=True): """ Get a serialized copy of the sync table. :param incoming: if True, get the last incoming sync, otherwise get the last outgoing sync :returns: list of {'remote_id', 'sync_point'} """ with self.get() as conn: curs = conn.execute(''' SELECT remote_id, sync_point FROM %s_sync ''' % 'incoming' if incoming else 'outgoing') result = [] for row in curs: result.append({'remote_id': row[0], 'sync_point': row[1]}) return result def get_replication_info(self): """ Get information about the DB required for replication. :returns: dict containing keys: hash, id, created_at, put_timestamp, delete_timestamp, count, max_row, and metadata """ self._commit_puts_stale_ok() query_part1 = ''' SELECT hash, id, created_at, put_timestamp, delete_timestamp, %s_count AS count, CASE WHEN SQLITE_SEQUENCE.seq IS NOT NULL THEN SQLITE_SEQUENCE.seq ELSE -1 END AS max_row, ''' % \ self.db_contains_type query_part2 = ''' FROM (%s_stat LEFT JOIN SQLITE_SEQUENCE ON SQLITE_SEQUENCE.name == '%s') LIMIT 1 ''' % (self.db_type, self.db_contains_type) with self.get() as conn: try: curs = conn.execute(query_part1 + 'metadata' + query_part2) except sqlite3.OperationalError as err: if 'no such column: metadata' not in str(err): raise curs = conn.execute(query_part1 + "'' as metadata" + query_part2) curs.row_factory = dict_factory return curs.fetchone() def _commit_puts(self, item_list=None): """ Scan for .pending files and commit the found records by feeding them to merge_items(). :param item_list: A list of items to commit in addition to .pending """ if self.db_file == ':memory:' or not os.path.exists(self.pending_file): return if item_list is None: item_list = [] with lock_parent_directory(self.pending_file, self.pending_timeout): self._preallocate() if not os.path.getsize(self.pending_file): if item_list: self.merge_items(item_list) return with open(self.pending_file, 'r+b') as fp: for entry in fp.read().split(':'): if entry: try: self._commit_puts_load(item_list, entry) except Exception: self.logger.exception( _('Invalid pending entry %(file)s: %(entry)s'), {'file': self.pending_file, 'entry': entry}) if item_list: self.merge_items(item_list) try: os.ftruncate(fp.fileno(), 0) except OSError as err: if err.errno != errno.ENOENT: raise def _commit_puts_stale_ok(self): """ Catch failures of _commit_puts() if broker is intended for reading of stats, and thus does not care for pending updates. """ try: self._commit_puts() except LockTimeout: if not self.stale_reads_ok: raise def _commit_puts_load(self, item_list, entry): """ Unmarshall the :param:entry and append it to :param:item_list. This is implemented by a particular broker to be compatible with its :func:`merge_items`. """ raise NotImplementedError def merge_syncs(self, sync_points, incoming=True): """ Merge a list of sync points with the incoming sync table. :param sync_points: list of sync points where a sync point is a dict of {'sync_point', 'remote_id'} :param incoming: if True, get the last incoming sync, otherwise get the last outgoing sync """ with self.get() as conn: for rec in sync_points: try: conn.execute(''' INSERT INTO %s_sync (sync_point, remote_id) VALUES (?, ?) ''' % ('incoming' if incoming else 'outgoing'), (rec['sync_point'], rec['remote_id'])) except sqlite3.IntegrityError: conn.execute(''' UPDATE %s_sync SET sync_point=max(?, sync_point) WHERE remote_id=? ''' % ('incoming' if incoming else 'outgoing'), (rec['sync_point'], rec['remote_id'])) conn.commit() def _preallocate(self): """ The idea is to allocate space in front of an expanding db. If it gets within 512k of a boundary, it allocates to the next boundary. Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after. """ if not DB_PREALLOCATION or self.db_file == ':memory:': return MB = (1024 * 1024) def prealloc_points(): for pm in (1, 2, 5, 10, 25, 50): yield pm * MB while True: pm += 50 yield pm * MB stat = os.stat(self.db_file) file_size = stat.st_size allocated_size = stat.st_blocks * 512 for point in prealloc_points(): if file_size <= point - MB / 2: prealloc_size = point break if allocated_size < prealloc_size: with open(self.db_file, 'rb+') as fp: fallocate(fp.fileno(), int(prealloc_size)) @property def metadata(self): """ Returns the metadata dict for the database. The metadata dict values are tuples of (value, timestamp) where the timestamp indicates when that key was set to that value. """ with self.get() as conn: try: metadata = conn.execute('SELECT metadata FROM %s_stat' % self.db_type).fetchone()[0] except sqlite3.OperationalError as err: if 'no such column: metadata' not in str(err): raise metadata = '' if metadata: metadata = json.loads(metadata) utf8encodekeys(metadata) else: metadata = {} return metadata def update_metadata(self, metadata_updates): """ Updates the metadata dict for the database. The metadata dict values are tuples of (value, timestamp) where the timestamp indicates when that key was set to that value. Key/values will only be overwritten if the timestamp is newer. To delete a key, set its value to ('', timestamp). These empty keys will eventually be removed by :func:`reclaim` """ old_metadata = self.metadata if set(metadata_updates).issubset(set(old_metadata)): for key, (value, timestamp) in metadata_updates.iteritems(): if timestamp > old_metadata[key][1]: break else: return with self.get() as conn: try: md = conn.execute('SELECT metadata FROM %s_stat' % self.db_type).fetchone()[0] md = json.loads(md) if md else {} utf8encodekeys(md) except sqlite3.OperationalError as err: if 'no such column: metadata' not in str(err): raise conn.execute(""" ALTER TABLE %s_stat ADD COLUMN metadata TEXT DEFAULT '' """ % self.db_type) md = {} for key, value_timestamp in metadata_updates.iteritems(): value, timestamp = value_timestamp if key not in md or timestamp > md[key][1]: md[key] = value_timestamp conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type, (json.dumps(md),)) conn.commit() def reclaim(self, age_timestamp, sync_timestamp): """ Delete rows from the db_contains_type table that are marked deleted and whose created_at timestamp is < age_timestamp. Also deletes rows from incoming_sync and outgoing_sync where the updated_at timestamp is < sync_timestamp. In addition, this calls the DatabaseBroker's :func:`_reclaim` method. :param age_timestamp: max created_at timestamp of object rows to delete :param sync_timestamp: max update_at timestamp of sync rows to delete """ self._commit_puts() with self.get() as conn: conn.execute(''' DELETE FROM %s WHERE deleted = 1 AND %s < ? ''' % (self.db_contains_type, self.db_reclaim_timestamp), (age_timestamp,)) try: conn.execute(''' DELETE FROM outgoing_sync WHERE updated_at < ? ''', (sync_timestamp,)) conn.execute(''' DELETE FROM incoming_sync WHERE updated_at < ? ''', (sync_timestamp,)) except sqlite3.OperationalError as err: # Old dbs didn't have updated_at in the _sync tables. if 'no such column: updated_at' not in str(err): raise DatabaseBroker._reclaim(self, conn, age_timestamp) conn.commit() def _reclaim(self, conn, timestamp): """ Removes any empty metadata values older than the timestamp using the given database connection. This function will not call commit on the conn, but will instead return True if the database needs committing. This function was created as a worker to limit transactions and commits from other related functions. :param conn: Database connection to reclaim metadata within. :param timestamp: Empty metadata items last updated before this timestamp will be removed. :returns: True if conn.commit() should be called """ try: md = conn.execute('SELECT metadata FROM %s_stat' % self.db_type).fetchone()[0] if md: md = json.loads(md) keys_to_delete = [] for key, (value, value_timestamp) in md.iteritems(): if value == '' and value_timestamp < timestamp: keys_to_delete.append(key) if keys_to_delete: for key in keys_to_delete: del md[key] conn.execute('UPDATE %s_stat SET metadata = ?' % self.db_type, (json.dumps(md),)) return True except sqlite3.OperationalError as err: if 'no such column: metadata' not in str(err): raise return False def update_put_timestamp(self, timestamp): """ Update the put_timestamp. Only modifies it if it is greater than the current timestamp. :param timestamp: put timestamp """ with self.get() as conn: conn.execute( 'UPDATE %s_stat SET put_timestamp = ?' ' WHERE put_timestamp < ?' % self.db_type, (timestamp, timestamp)) conn.commit() swift-1.13.1/swift/common/ring/0000775000175400017540000000000012323703665017461 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/common/ring/utils.py0000664000175400017540000003024012323703611021161 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from collections import defaultdict import optparse def tiers_for_dev(dev): """ Returns a tuple of tiers for a given device in ascending order by length. :returns: tuple of tiers """ t1 = dev['region'] t2 = dev['zone'] t3 = "{ip}:{port}".format(ip=dev.get('ip'), port=dev.get('port')) t4 = dev['id'] return ((t1,), (t1, t2), (t1, t2, t3), (t1, t2, t3, t4)) def build_tier_tree(devices): """ Construct the tier tree from the zone layout. The tier tree is a dictionary that maps tiers to their child tiers. A synthetic root node of () is generated so that there's one tree, not a forest. Example: region 1 -+---- zone 1 -+---- 192.168.101.1:6000 -+---- device id 0 | | | | | +---- device id 1 | | | | | +---- device id 2 | | | +---- 192.168.101.2:6000 -+---- device id 3 | | | +---- device id 4 | | | +---- device id 5 | +---- zone 2 -+---- 192.168.102.1:6000 -+---- device id 6 | | | +---- device id 7 | | | +---- device id 8 | +---- 192.168.102.2:6000 -+---- device id 9 | +---- device id 10 region 2 -+---- zone 1 -+---- 192.168.201.1:6000 -+---- device id 12 | | | +---- device id 13 | | | +---- device id 14 | +---- 192.168.201.2:6000 -+---- device id 15 | +---- device id 16 | +---- device id 17 The tier tree would look like: { (): [(1,), (2,)], (1,): [(1, 1), (1, 2)], (2,): [(2, 1)], (1, 1): [(1, 1, 192.168.101.1:6000), (1, 1, 192.168.101.2:6000)], (1, 2): [(1, 2, 192.168.102.1:6000), (1, 2, 192.168.102.2:6000)], (2, 1): [(2, 1, 192.168.201.1:6000), (2, 1, 192.168.201.2:6000)], (1, 1, 192.168.101.1:6000): [(1, 1, 192.168.101.1:6000, 0), (1, 1, 192.168.101.1:6000, 1), (1, 1, 192.168.101.1:6000, 2)], (1, 1, 192.168.101.2:6000): [(1, 1, 192.168.101.2:6000, 3), (1, 1, 192.168.101.2:6000, 4), (1, 1, 192.168.101.2:6000, 5)], (1, 2, 192.168.102.1:6000): [(1, 2, 192.168.102.1:6000, 6), (1, 2, 192.168.102.1:6000, 7), (1, 2, 192.168.102.1:6000, 8)], (1, 2, 192.168.102.2:6000): [(1, 2, 192.168.102.2:6000, 9), (1, 2, 192.168.102.2:6000, 10)], (2, 1, 192.168.201.1:6000): [(2, 1, 192.168.201.1:6000, 12), (2, 1, 192.168.201.1:6000, 13), (2, 1, 192.168.201.1:6000, 14)], (2, 1, 192.168.201.2:6000): [(2, 1, 192.168.201.2:6000, 15), (2, 1, 192.168.201.2:6000, 16), (2, 1, 192.168.201.2:6000, 17)], } :devices: device dicts from which to generate the tree :returns: tier tree """ tier2children = defaultdict(set) for dev in devices: for tier in tiers_for_dev(dev): if len(tier) > 1: tier2children[tier[0:-1]].add(tier) else: tier2children[()].add(tier) return tier2children def parse_search_value(search_value): """The can be of the form:: drz-:[R:]/ _ Where and are replication ip and port. Any part is optional, but you must include at least one part. Examples:: d74 Matches the device id 74 r4 Matches devices in region 4 z1 Matches devices in zone 1 z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4 1.2.3.4 Matches devices in any zone with the ip 1.2.3.4 z1:5678 Matches devices in zone 1 using port 5678 :5678 Matches devices that use port 5678 R5.6.7.8 Matches devices that use replication ip 5.6.7.8 R:5678 Matches devices that use replication port 5678 1.2.3.4R5.6.7.8 Matches devices that use ip 1.2.3.4 and replication ip 5.6.7.8 /sdb1 Matches devices with the device name sdb1 _shiny Matches devices with shiny in the meta data _"snet: 5.6.7.8" Matches devices with snet: 5.6.7.8 in the meta data [::1] Matches devices in any zone with the ip ::1 z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678 Most specific example:: d74r4z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" Nerd explanation: All items require their single character prefix except the ip, in which case the - is optional unless the device id or zone is also included. """ orig_search_value = search_value match = {} if search_value.startswith('d'): i = 1 while i < len(search_value) and search_value[i].isdigit(): i += 1 match['id'] = int(search_value[1:i]) search_value = search_value[i:] if search_value.startswith('r'): i = 1 while i < len(search_value) and search_value[i].isdigit(): i += 1 match['region'] = int(search_value[1:i]) search_value = search_value[i:] if search_value.startswith('z'): i = 1 while i < len(search_value) and search_value[i].isdigit(): i += 1 match['zone'] = int(search_value[1:i]) search_value = search_value[i:] if search_value.startswith('-'): search_value = search_value[1:] if len(search_value) and search_value[0].isdigit(): i = 1 while i < len(search_value) and search_value[i] in '0123456789.': i += 1 match['ip'] = search_value[:i] search_value = search_value[i:] elif len(search_value) and search_value[0] == '[': i = 1 while i < len(search_value) and search_value[i] != ']': i += 1 i += 1 match['ip'] = search_value[:i].lstrip('[').rstrip(']') search_value = search_value[i:] if search_value.startswith(':'): i = 1 while i < len(search_value) and search_value[i].isdigit(): i += 1 match['port'] = int(search_value[1:i]) search_value = search_value[i:] # replication parameters if search_value.startswith('R'): search_value = search_value[1:] if len(search_value) and search_value[0].isdigit(): i = 1 while (i < len(search_value) and search_value[i] in '0123456789.'): i += 1 match['replication_ip'] = search_value[:i] search_value = search_value[i:] elif len(search_value) and search_value[0] == '[': i = 1 while i < len(search_value) and search_value[i] != ']': i += 1 i += 1 match['replication_ip'] = search_value[:i].lstrip('[').rstrip(']') search_value = search_value[i:] if search_value.startswith(':'): i = 1 while i < len(search_value) and search_value[i].isdigit(): i += 1 match['replication_port'] = int(search_value[1:i]) search_value = search_value[i:] if search_value.startswith('/'): i = 1 while i < len(search_value) and search_value[i] != '_': i += 1 match['device'] = search_value[1:i] search_value = search_value[i:] if search_value.startswith('_'): match['meta'] = search_value[1:] search_value = '' if search_value: raise ValueError('Invalid : %s' % repr(orig_search_value)) return match def parse_args(argvish): """ Build OptionParser and evaluate command line arguments. """ parser = optparse.OptionParser() parser.add_option('-r', '--region', type="int", help="Region") parser.add_option('-z', '--zone', type="int", help="Zone") parser.add_option('-i', '--ip', type="string", help="IP address") parser.add_option('-p', '--port', type="int", help="Port number") parser.add_option('-j', '--replication-ip', type="string", help="Replication IP address") parser.add_option('-q', '--replication-port', type="int", help="Replication port number") parser.add_option('-d', '--device', type="string", help="Device name (e.g. md0, sdb1)") parser.add_option('-w', '--weight', type="float", help="Device weight") parser.add_option('-m', '--meta', type="string", default="", help="Extra device info (just a string)") return parser.parse_args(argvish) def parse_builder_ring_filename_args(argvish): first_arg = argvish[1] if first_arg.endswith('.ring.gz'): ring_file = first_arg builder_file = first_arg[:-len('.ring.gz')] + '.builder' else: builder_file = first_arg if not builder_file.endswith('.builder'): ring_file = first_arg else: ring_file = builder_file[:-len('.builder')] if not first_arg.endswith('.ring.gz'): ring_file += '.ring.gz' return builder_file, ring_file def build_dev_from_opts(opts): """ Convert optparse stype options into a device dictionary. """ for attribute, shortopt, longopt in (['region', '-r', '--region'], ['zone', '-z', '--zone'], ['ip', '-i', '--ip'], ['port', '-p', '--port'], ['device', '-d', '--device'], ['weight', '-w', '--weight']): if not getattr(opts, attribute, None): raise ValueError('Required argument %s/%s not specified.' % (shortopt, longopt)) replication_ip = opts.replication_ip or opts.ip replication_port = opts.replication_port or opts.port return {'region': opts.region, 'zone': opts.zone, 'ip': opts.ip, 'port': opts.port, 'device': opts.device, 'meta': opts.meta, 'replication_ip': replication_ip, 'replication_port': replication_port, 'weight': opts.weight} swift-1.13.1/swift/common/ring/ring.py0000664000175400017540000004135312323703614020772 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import array import cPickle as pickle from collections import defaultdict from gzip import GzipFile from os.path import getmtime import struct from time import time import os from io import BufferedReader from hashlib import md5 from itertools import chain from tempfile import NamedTemporaryFile from swift.common.utils import hash_path, validate_configuration, json from swift.common.ring.utils import tiers_for_dev class RingData(object): """Partitioned consistent hashing ring data (used for serialization).""" def __init__(self, replica2part2dev_id, devs, part_shift): self.devs = devs self._replica2part2dev_id = replica2part2dev_id self._part_shift = part_shift for dev in self.devs: if dev is not None: dev.setdefault("region", 1) @classmethod def deserialize_v1(cls, gz_file): json_len, = struct.unpack('!I', gz_file.read(4)) ring_dict = json.loads(gz_file.read(json_len)) ring_dict['replica2part2dev_id'] = [] partition_count = 1 << (32 - ring_dict['part_shift']) for x in xrange(ring_dict['replica_count']): ring_dict['replica2part2dev_id'].append( array.array('H', gz_file.read(2 * partition_count))) return ring_dict @classmethod def load(cls, filename): """ Load ring data from a file. :param filename: Path to a file serialized by the save() method. :returns: A RingData instance containing the loaded data. """ gz_file = GzipFile(filename, 'rb') # Python 2.6 GzipFile doesn't support BufferedIO if hasattr(gz_file, '_checkReadable'): gz_file = BufferedReader(gz_file) # See if the file is in the new format magic = gz_file.read(4) if magic == 'R1NG': version, = struct.unpack('!H', gz_file.read(2)) if version == 1: ring_data = cls.deserialize_v1(gz_file) else: raise Exception('Unknown ring format version %d' % version) else: # Assume old-style pickled ring gz_file.seek(0) ring_data = pickle.load(gz_file) if not hasattr(ring_data, 'devs'): ring_data = RingData(ring_data['replica2part2dev_id'], ring_data['devs'], ring_data['part_shift']) return ring_data def serialize_v1(self, file_obj): # Write out new-style serialization magic and version: file_obj.write(struct.pack('!4sH', 'R1NG', 1)) ring = self.to_dict() json_encoder = json.JSONEncoder(sort_keys=True) json_text = json_encoder.encode( {'devs': ring['devs'], 'part_shift': ring['part_shift'], 'replica_count': len(ring['replica2part2dev_id'])}) json_len = len(json_text) file_obj.write(struct.pack('!I', json_len)) file_obj.write(json_text) for part2dev_id in ring['replica2part2dev_id']: file_obj.write(part2dev_id.tostring()) def save(self, filename): """ Serialize this RingData instance to disk. :param filename: File into which this instance should be serialized. """ # Override the timestamp so that the same ring data creates # the same bytes on disk. This makes a checksum comparison a # good way to see if two rings are identical. # # This only works on Python 2.7; on 2.6, we always get the # current time in the gzip output. tempf = NamedTemporaryFile(dir=".", prefix=filename, delete=False) try: gz_file = GzipFile(filename, mode='wb', fileobj=tempf, mtime=1300507380.0) except TypeError: gz_file = GzipFile(filename, mode='wb', fileobj=tempf) self.serialize_v1(gz_file) gz_file.close() tempf.flush() os.fsync(tempf.fileno()) tempf.close() os.rename(tempf.name, filename) def to_dict(self): return {'devs': self.devs, 'replica2part2dev_id': self._replica2part2dev_id, 'part_shift': self._part_shift} class Ring(object): """ Partitioned consistent hashing ring. :param serialized_path: path to serialized RingData instance :param reload_time: time interval in seconds to check for a ring change """ def __init__(self, serialized_path, reload_time=15, ring_name=None): # can't use the ring unless HASH_PATH_SUFFIX is set validate_configuration() if ring_name: self.serialized_path = os.path.join(serialized_path, ring_name + '.ring.gz') else: self.serialized_path = os.path.join(serialized_path) self.reload_time = reload_time self._reload(force=True) def _reload(self, force=False): self._rtime = time() + self.reload_time if force or self.has_changed(): ring_data = RingData.load(self.serialized_path) self._mtime = getmtime(self.serialized_path) self._devs = ring_data.devs # NOTE(akscram): Replication parameters like replication_ip # and replication_port are required for # replication process. An old replication # ring doesn't contain this parameters into # device. Old-style pickled rings won't have # region information. for dev in self._devs: if dev: dev.setdefault('region', 1) if 'ip' in dev: dev.setdefault('replication_ip', dev['ip']) if 'port' in dev: dev.setdefault('replication_port', dev['port']) self._replica2part2dev_id = ring_data._replica2part2dev_id self._part_shift = ring_data._part_shift self._rebuild_tier_data() # Do this now, when we know the data has changed, rather then # doing it on every call to get_more_nodes(). regions = set() zones = set() ip_ports = set() self._num_devs = 0 for dev in self._devs: if dev: regions.add(dev['region']) zones.add((dev['region'], dev['zone'])) ip_ports.add((dev['region'], dev['zone'], dev['ip'], dev['port'])) self._num_devs += 1 self._num_regions = len(regions) self._num_zones = len(zones) self._num_ip_ports = len(ip_ports) def _rebuild_tier_data(self): self.tier2devs = defaultdict(list) for dev in self._devs: if not dev: continue for tier in tiers_for_dev(dev): self.tier2devs[tier].append(dev) tiers_by_length = defaultdict(list) for tier in self.tier2devs: tiers_by_length[len(tier)].append(tier) self.tiers_by_length = sorted(tiers_by_length.values(), key=lambda x: len(x[0])) for tiers in self.tiers_by_length: tiers.sort() @property def replica_count(self): """Number of replicas (full or partial) used in the ring.""" return len(self._replica2part2dev_id) @property def partition_count(self): """Number of partitions in the ring.""" return len(self._replica2part2dev_id[0]) @property def devs(self): """devices in the ring""" if time() > self._rtime: self._reload() return self._devs def has_changed(self): """ Check to see if the ring on disk is different than the current one in memory. :returns: True if the ring on disk has changed, False otherwise """ return getmtime(self.serialized_path) != self._mtime def _get_part_nodes(self, part): part_nodes = [] seen_ids = set() for r2p2d in self._replica2part2dev_id: if part < len(r2p2d): dev_id = r2p2d[part] if dev_id not in seen_ids: part_nodes.append(self.devs[dev_id]) seen_ids.add(dev_id) return part_nodes def get_part(self, account, container=None, obj=None): """ Get the partition for an account/container/object. :param account: account name :param container: container name :param obj: object name :returns: the partition number """ key = hash_path(account, container, obj, raw_digest=True) if time() > self._rtime: self._reload() part = struct.unpack_from('>I', key)[0] >> self._part_shift return part def get_part_nodes(self, part): """ Get the nodes that are responsible for the partition. If one node is responsible for more than one replica of the same partition, it will only appear in the output once. :param part: partition to get nodes for :returns: list of node dicts See :func:`get_nodes` for a description of the node dicts. """ if time() > self._rtime: self._reload() return self._get_part_nodes(part) def get_nodes(self, account, container=None, obj=None): """ Get the partition and nodes for an account/container/object. If a node is responsible for more than one replica, it will only appear in the output once. :param account: account name :param container: container name :param obj: object name :returns: a tuple of (partition, list of node dicts) Each node dict will have at least the following keys: ====== =============================================================== id unique integer identifier amongst devices weight a float of the relative weight of this device as compared to others; this indicates how many partitions the builder will try to assign to this device zone integer indicating which zone the device is in; a given partition will not be assigned to multiple devices within the same zone ip the ip address of the device port the tcp port of the device device the device's name on disk (sdb1, for example) meta general use 'extra' field; for example: the online date, the hardware description ====== =============================================================== """ part = self.get_part(account, container, obj) return part, self._get_part_nodes(part) def get_more_nodes(self, part): """ Generator to get extra nodes for a partition for hinted handoff. The handoff nodes will try to be in zones other than the primary zones, will take into account the device weights, and will usually keep the same sequences of handoffs even with ring changes. :param part: partition to get handoff nodes for :returns: generator of node dicts See :func:`get_nodes` for a description of the node dicts. """ if time() > self._rtime: self._reload() primary_nodes = self._get_part_nodes(part) used = set(d['id'] for d in primary_nodes) same_regions = set(d['region'] for d in primary_nodes) same_zones = set((d['region'], d['zone']) for d in primary_nodes) same_ip_ports = set((d['region'], d['zone'], d['ip'], d['port']) for d in primary_nodes) parts = len(self._replica2part2dev_id[0]) start = struct.unpack_from( '>I', md5(str(part)).digest())[0] >> self._part_shift inc = int(parts / 65536) or 1 # Multiple loops for execution speed; the checks and bookkeeping get # simpler as you go along hit_all_regions = len(same_regions) == self._num_regions for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): if hit_all_regions: # At this point, there are no regions left untouched, so we # can stop looking. break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] dev = self._devs[dev_id] region = dev['region'] if dev_id not in used and region not in same_regions: yield dev used.add(dev_id) same_regions.add(region) zone = dev['zone'] ip_port = (region, zone, dev['ip'], dev['port']) same_zones.add((region, zone)) same_ip_ports.add(ip_port) if len(same_regions) == self._num_regions: hit_all_regions = True break hit_all_zones = len(same_zones) == self._num_zones for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): if hit_all_zones: # Much like we stopped looking for fresh regions before, we # can now stop looking for fresh zones; there are no more. break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] dev = self._devs[dev_id] zone = (dev['region'], dev['zone']) if dev_id not in used and zone not in same_zones: yield dev used.add(dev_id) same_zones.add(zone) ip_port = zone + (dev['ip'], dev['port']) same_ip_ports.add(ip_port) if len(same_zones) == self._num_zones: hit_all_zones = True break hit_all_ip_ports = len(same_ip_ports) == self._num_ip_ports for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): if hit_all_ip_ports: # We've exhausted the pool of unused backends, so stop # looking. break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] dev = self._devs[dev_id] ip_port = (dev['region'], dev['zone'], dev['ip'], dev['port']) if dev_id not in used and ip_port not in same_ip_ports: yield dev used.add(dev_id) same_ip_ports.add(ip_port) if len(same_ip_ports) == self._num_ip_ports: hit_all_ip_ports = True break hit_all_devs = len(used) == self._num_devs for handoff_part in chain(xrange(start, parts, inc), xrange(inc - ((parts - start) % inc), start, inc)): if hit_all_devs: # We've used every device we have, so let's stop looking for # unused devices now. break for part2dev_id in self._replica2part2dev_id: if handoff_part < len(part2dev_id): dev_id = part2dev_id[handoff_part] if dev_id not in used: yield self._devs[dev_id] used.add(dev_id) if len(used) == self._num_devs: hit_all_devs = True break swift-1.13.1/swift/common/ring/builder.py0000664000175400017540000013132112323703611021451 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import bisect import itertools import math import random import cPickle as pickle from array import array from collections import defaultdict from time import time from swift.common import exceptions from swift.common.ring import RingData from swift.common.ring.utils import tiers_for_dev, build_tier_tree MAX_BALANCE = 999.99 class RingBuilder(object): """ Used to build swift.common.ring.RingData instances to be written to disk and used with swift.common.ring.Ring instances. See bin/swift-ring-builder for example usage. The instance variable devs_changed indicates if the device information has changed since the last balancing. This can be used by tools to know whether a rebalance request is an isolated request or due to added, changed, or removed devices. :param part_power: number of partitions = 2**part_power. :param replicas: number of replicas for each partition :param min_part_hours: minimum number of hours between partition changes """ def __init__(self, part_power, replicas, min_part_hours): if part_power > 32: raise ValueError("part_power must be at most 32 (was %d)" % (part_power,)) if replicas < 1: raise ValueError("replicas must be at least 1 (was %.6f)" % (replicas,)) if min_part_hours < 0: raise ValueError("min_part_hours must be non-negative (was %d)" % (min_part_hours,)) self.part_power = part_power self.replicas = replicas self.min_part_hours = min_part_hours self.parts = 2 ** self.part_power self.devs = [] self.devs_changed = False self.version = 0 # _replica2part2dev maps from replica number to partition number to # device id. So, for a three replica, 2**23 ring, it's an array of # three 2**23 arrays of device ids (unsigned shorts). This can work a # bit faster than the 2**23 array of triplet arrays of device ids in # many circumstances. Making one big 2**23 * 3 array didn't seem to # have any speed change; though you're welcome to try it again (it was # a while ago, code-wise, when I last tried it). self._replica2part2dev = None # _last_part_moves is a 2**23 array of unsigned bytes representing the # number of hours since a given partition was last moved. This is used # to guarantee we don't move a partition twice within a given number of # hours (24 is my usual test). Removing a device or setting its weight # to 0 overrides this behavior as it's assumed those actions are done # because of device failure. # _last_part_moves_epoch indicates the time the offsets in # _last_part_moves is based on. self._last_part_moves_epoch = None self._last_part_moves = None self._last_part_gather_start = 0 self._remove_devs = [] self._ring = None def weight_of_one_part(self): """ Returns the weight of each partition as calculated from the total weight of all the devices. """ try: return self.parts * self.replicas / \ sum(d['weight'] for d in self._iter_devs()) except ZeroDivisionError: raise exceptions.EmptyRingError('There are no devices in this ' 'ring, or all devices have been ' 'deleted') def copy_from(self, builder): """ Reinitializes this RingBuilder instance from data obtained from the builder dict given. Code example:: b = RingBuilder(1, 1, 1) # Dummy values b.copy_from(builder) This is to restore a RingBuilder that has had its b.to_dict() previously saved. """ if hasattr(builder, 'devs'): self.part_power = builder.part_power self.replicas = builder.replicas self.min_part_hours = builder.min_part_hours self.parts = builder.parts self.devs = builder.devs self.devs_changed = builder.devs_changed self.version = builder.version self._replica2part2dev = builder._replica2part2dev self._last_part_moves_epoch = builder._last_part_moves_epoch self._last_part_moves = builder._last_part_moves self._last_part_gather_start = builder._last_part_gather_start self._remove_devs = builder._remove_devs else: self.part_power = builder['part_power'] self.replicas = builder['replicas'] self.min_part_hours = builder['min_part_hours'] self.parts = builder['parts'] self.devs = builder['devs'] self.devs_changed = builder['devs_changed'] self.version = builder['version'] self._replica2part2dev = builder['_replica2part2dev'] self._last_part_moves_epoch = builder['_last_part_moves_epoch'] self._last_part_moves = builder['_last_part_moves'] self._last_part_gather_start = builder['_last_part_gather_start'] self._remove_devs = builder['_remove_devs'] self._ring = None # Old builders may not have a region defined for their devices, in # which case we default it to 1. for dev in self._iter_devs(): dev.setdefault("region", 1) def to_dict(self): """ Returns a dict that can be used later with copy_from to restore a RingBuilder. swift-ring-builder uses this to pickle.dump the dict to a file and later load that dict into copy_from. """ return {'part_power': self.part_power, 'replicas': self.replicas, 'min_part_hours': self.min_part_hours, 'parts': self.parts, 'devs': self.devs, 'devs_changed': self.devs_changed, 'version': self.version, '_replica2part2dev': self._replica2part2dev, '_last_part_moves_epoch': self._last_part_moves_epoch, '_last_part_moves': self._last_part_moves, '_last_part_gather_start': self._last_part_gather_start, '_remove_devs': self._remove_devs} def change_min_part_hours(self, min_part_hours): """ Changes the value used to decide if a given partition can be moved again. This restriction is to give the overall system enough time to settle a partition to its new location before moving it to yet another location. While no data would be lost if a partition is moved several times quickly, it could make that data unreachable for a short period of time. This should be set to at least the average full partition replication time. Starting it at 24 hours and then lowering it to what the replicator reports as the longest partition cycle is best. :param min_part_hours: new value for min_part_hours """ self.min_part_hours = min_part_hours def set_replicas(self, new_replica_count): """ Changes the number of replicas in this ring. If the new replica count is sufficiently different that self._replica2part2dev will change size, sets self.devs_changed. This is so tools like bin/swift-ring-builder can know to write out the new ring rather than bailing out due to lack of balance change. """ old_slots_used = int(self.parts * self.replicas) new_slots_used = int(self.parts * new_replica_count) if old_slots_used != new_slots_used: self.devs_changed = True self.replicas = new_replica_count def get_ring(self): """ Get the ring, or more specifically, the swift.common.ring.RingData. This ring data is the minimum required for use of the ring. The ring builder itself keeps additional data such as when partitions were last moved. """ # We cache the self._ring value so multiple requests for it don't build # it multiple times. Be sure to set self._ring = None whenever the ring # will need to be rebuilt. if not self._ring: # Make devs list (with holes for deleted devices) and not including # builder-specific extra attributes. devs = [None] * len(self.devs) for dev in self._iter_devs(): devs[dev['id']] = dict((k, v) for k, v in dev.items() if k not in ('parts', 'parts_wanted')) # Copy over the replica+partition->device assignments, the device # information, and the part_shift value (the number of bits to # shift an unsigned int >I right to obtain the partition for the # int). if not self._replica2part2dev: self._ring = RingData([], devs, 32 - self.part_power) else: self._ring = \ RingData([array('H', p2d) for p2d in self._replica2part2dev], devs, 32 - self.part_power) return self._ring def add_dev(self, dev): """ Add a device to the ring. This device dict should have a minimum of the following keys: ====== =============================================================== id unique integer identifier amongst devices. Defaults to the next id if the 'id' key is not provided in the dict weight a float of the relative weight of this device as compared to others; this indicates how many partitions the builder will try to assign to this device region integer indicating which region the device is in zone integer indicating which zone the device is in; a given partition will not be assigned to multiple devices within the same (region, zone) pair if there is any alternative ip the ip address of the device port the tcp port of the device device the device's name on disk (sdb1, for example) meta general use 'extra' field; for example: the online date, the hardware description ====== =============================================================== .. note:: This will not rebalance the ring immediately as you may want to make multiple changes for a single rebalance. :param dev: device dict :returns: id of device """ if 'id' not in dev: dev['id'] = 0 if self.devs: dev['id'] = max(d['id'] for d in self.devs if d) + 1 if dev['id'] < len(self.devs) and self.devs[dev['id']] is not None: raise exceptions.DuplicateDeviceError( 'Duplicate device id: %d' % dev['id']) # Add holes to self.devs to ensure self.devs[dev['id']] will be the dev while dev['id'] >= len(self.devs): self.devs.append(None) dev['weight'] = float(dev['weight']) dev['parts'] = 0 self.devs[dev['id']] = dev self._set_parts_wanted() self.devs_changed = True self.version += 1 return dev['id'] def set_dev_weight(self, dev_id, weight): """ Set the weight of a device. This should be called rather than just altering the weight key in the device dict directly, as the builder will need to rebuild some internal state to reflect the change. .. note:: This will not rebalance the ring immediately as you may want to make multiple changes for a single rebalance. :param dev_id: device id :param weight: new weight for device """ self.devs[dev_id]['weight'] = weight self._set_parts_wanted() self.devs_changed = True self.version += 1 def remove_dev(self, dev_id): """ Remove a device from the ring. .. note:: This will not rebalance the ring immediately as you may want to make multiple changes for a single rebalance. :param dev_id: device id """ dev = self.devs[dev_id] dev['weight'] = 0 self._remove_devs.append(dev) self._set_parts_wanted() self.devs_changed = True self.version += 1 def rebalance(self, seed=None): """ Rebalance the ring. This is the main work function of the builder, as it will assign and reassign partitions to devices in the ring based on weights, distinct zones, recent reassignments, etc. The process doesn't always perfectly assign partitions (that'd take a lot more analysis and therefore a lot more time -- I had code that did that before). Because of this, it keeps rebalancing until the device skew (number of partitions a device wants compared to what it has) gets below 1% or doesn't change by more than 1% (only happens with ring that can't be balanced no matter what -- like with 3 zones of differing weights with replicas set to 3). :returns: (number_of_partitions_altered, resulting_balance) """ if seed is not None: random.seed(seed) self._ring = None if self._last_part_moves_epoch is None: self._initial_balance() self.devs_changed = False return self.parts, self.get_balance() retval = 0 self._update_last_part_moves() last_balance = 0 new_parts, removed_part_count = self._adjust_replica2part2dev_size() retval += removed_part_count self._reassign_parts(new_parts) retval += len(new_parts) while True: reassign_parts = self._gather_reassign_parts() self._reassign_parts(reassign_parts) retval += len(reassign_parts) while self._remove_devs: self.devs[self._remove_devs.pop()['id']] = None balance = self.get_balance() if balance < 1 or abs(last_balance - balance) < 1 or \ retval == self.parts: break last_balance = balance self.devs_changed = False self.version += 1 return retval, balance def validate(self, stats=False): """ Validate the ring. This is a safety function to try to catch any bugs in the building process. It ensures partitions have been assigned to real devices, aren't doubly assigned, etc. It can also optionally check the even distribution of partitions across devices. :param stats: if True, check distribution of partitions across devices :returns: if stats is True, a tuple of (device_usage, worst_stat), else (None, None). device_usage[dev_id] will equal the number of partitions assigned to that device. worst_stat will equal the number of partitions the worst device is skewed from the number it should have. :raises RingValidationError: problem was found with the ring. """ # "len" showed up in profiling, so it's just computed once. dev_len = len(self.devs) parts_on_devs = sum(d['parts'] for d in self._iter_devs()) if not self._replica2part2dev: raise exceptions.RingValidationError( '_replica2part2dev empty; did you forget to rebalance?') parts_in_map = sum(len(p2d) for p2d in self._replica2part2dev) if parts_on_devs != parts_in_map: raise exceptions.RingValidationError( 'All partitions are not double accounted for: %d != %d' % (parts_on_devs, parts_in_map)) if stats: # dev_usage[dev_id] will equal the number of partitions assigned to # that device. dev_usage = array('I', (0 for _junk in xrange(dev_len))) for part2dev in self._replica2part2dev: for dev_id in part2dev: dev_usage[dev_id] += 1 for part, replica in self._each_part_replica(): dev_id = self._replica2part2dev[replica][part] if dev_id >= dev_len or not self.devs[dev_id]: raise exceptions.RingValidationError( "Partition %d, replica %d was not allocated " "to a device." % (part, replica)) for dev in self._iter_devs(): if not isinstance(dev['port'], int): raise exceptions.RingValidationError( "Device %d has port %r, which is not an integer." % (dev['id'], dev['port'])) if stats: weight_of_one_part = self.weight_of_one_part() worst = 0 for dev in self._iter_devs(): if not dev['weight']: if dev_usage[dev['id']]: # If a device has no weight, but has partitions, then # its overage is considered "infinity" and therefore # always the worst possible. We show MAX_BALANCE for # convenience. worst = MAX_BALANCE break continue skew = abs(100.0 * dev_usage[dev['id']] / (dev['weight'] * weight_of_one_part) - 100.0) if skew > worst: worst = skew return dev_usage, worst return None, None def get_balance(self): """ Get the balance of the ring. The balance value is the highest percentage off the desired amount of partitions a given device wants. For instance, if the "worst" device wants (based on its weight relative to the sum of all the devices' weights) 123 partitions and it has 124 partitions, the balance value would be 0.83 (1 extra / 123 wanted * 100 for percentage). :returns: balance of the ring """ balance = 0 weight_of_one_part = self.weight_of_one_part() for dev in self._iter_devs(): if not dev['weight']: if dev['parts']: # If a device has no weight, but has partitions, then its # overage is considered "infinity" and therefore always the # worst possible. We show MAX_BALANCE for convenience. balance = MAX_BALANCE break continue dev_balance = abs(100.0 * dev['parts'] / (dev['weight'] * weight_of_one_part) - 100.0) if dev_balance > balance: balance = dev_balance return balance def pretend_min_part_hours_passed(self): """ Override min_part_hours by marking all partitions as having been moved 255 hours ago. This can be used to force a full rebalance on the next call to rebalance. """ for part in xrange(self.parts): self._last_part_moves[part] = 0xff def get_part_devices(self, part): """ Get the devices that are responsible for the partition, filtering out duplicates. :param part: partition to get devices for :returns: list of device dicts """ devices = [] for dev in self._devs_for_part(part): if dev not in devices: devices.append(dev) return devices def _iter_devs(self): """ Returns an iterator all the non-None devices in the ring. Note that this means list(b._iter_devs())[some_id] may not equal b.devs[some_id]; you will have to check the 'id' key of each device to obtain its dev_id. """ for dev in self.devs: if dev is not None: yield dev def _set_parts_wanted(self): """ Sets the parts_wanted key for each of the devices to the number of partitions the device wants based on its relative weight. This key is used to sort the devices according to "most wanted" during rebalancing to best distribute partitions. A negative parts_wanted indicates the device is "overweight" and wishes to give partitions away if possible. """ weight_of_one_part = self.weight_of_one_part() for dev in self._iter_devs(): if not dev['weight']: # With no weight, that means we wish to "drain" the device. So # we set the parts_wanted to a really large negative number to # indicate its strong desire to give up everything it has. dev['parts_wanted'] = -self.parts * self.replicas else: dev['parts_wanted'] = \ int(weight_of_one_part * dev['weight']) - dev['parts'] def _adjust_replica2part2dev_size(self): """ Make sure that the lengths of the arrays in _replica2part2dev are correct for the current value of self.replicas. Example: self.part_power = 8 self.replicas = 2.25 self._replica2part2dev will contain 3 arrays: the first 2 of length 256 (2**8), and the last of length 64 (0.25 * 2**8). Returns a 2-tuple: the first element is a list of (partition, replicas) tuples indicating which replicas need to be (re)assigned to devices, and the second element is a count of how many replicas were removed. """ removed_replicas = 0 fractional_replicas, whole_replicas = math.modf(self.replicas) whole_replicas = int(whole_replicas) desired_lengths = [self.parts] * whole_replicas if fractional_replicas: desired_lengths.append(int(self.parts * fractional_replicas)) to_assign = defaultdict(list) if self._replica2part2dev is not None: # If we crossed an integer threshold (say, 4.1 --> 4), # we'll have a partial extra replica clinging on here. Clean # up any such extra stuff. for part2dev in self._replica2part2dev[len(desired_lengths):]: for dev_id in part2dev: dev_losing_part = self.devs[dev_id] dev_losing_part['parts'] -= 1 removed_replicas += 1 self._replica2part2dev = \ self._replica2part2dev[:len(desired_lengths)] else: self._replica2part2dev = [] for replica, desired_length in enumerate(desired_lengths): if replica < len(self._replica2part2dev): part2dev = self._replica2part2dev[replica] if len(part2dev) < desired_length: # Not long enough: needs to be extended and the # newly-added pieces assigned to devices. for part in xrange(len(part2dev), desired_length): to_assign[part].append(replica) part2dev.append(0) elif len(part2dev) > desired_length: # Too long: truncate this mapping. for part in xrange(desired_length, len(part2dev)): dev_losing_part = self.devs[part2dev[part]] dev_losing_part['parts'] -= 1 removed_replicas += 1 self._replica2part2dev[replica] = part2dev[:desired_length] else: # Mapping not present at all: make one up and assign # all of it. for part in xrange(desired_length): to_assign[part].append(replica) self._replica2part2dev.append( array('H', (0 for _junk in xrange(desired_length)))) return (list(to_assign.iteritems()), removed_replicas) def _initial_balance(self): """ Initial partition assignment is the same as rebalancing an existing ring, but with some initial setup beforehand. """ self._last_part_moves = array('B', (0 for _junk in xrange(self.parts))) self._last_part_moves_epoch = int(time()) self._reassign_parts(self._adjust_replica2part2dev_size()[0]) def _update_last_part_moves(self): """ Updates how many hours ago each partition was moved based on the current time. The builder won't move a partition that has been moved more recently than min_part_hours. """ elapsed_hours = int(time() - self._last_part_moves_epoch) / 3600 for part in xrange(self.parts): # The "min(self._last_part_moves[part] + elapsed_hours, 0xff)" # which was here showed up in profiling, so it got inlined. last_plus_elapsed = self._last_part_moves[part] + elapsed_hours if last_plus_elapsed < 0xff: self._last_part_moves[part] = last_plus_elapsed else: self._last_part_moves[part] = 0xff self._last_part_moves_epoch = int(time()) def _gather_reassign_parts(self): """ Returns a list of (partition, replicas) pairs to be reassigned by gathering from removed devices, insufficiently-far-apart replicas, and overweight drives. """ # inline memoization of tiers_for_dev() results (profiling reveals it # as a hot-spot). tfd = {} # First we gather partitions from removed devices. Since removed # devices usually indicate device failures, we have no choice but to # reassign these partitions. However, we mark them as moved so later # choices will skip other replicas of the same partition if possible. removed_dev_parts = defaultdict(list) if self._remove_devs: dev_ids = [d['id'] for d in self._remove_devs if d['parts']] if dev_ids: for part, replica in self._each_part_replica(): dev_id = self._replica2part2dev[replica][part] if dev_id in dev_ids: self._last_part_moves[part] = 0 removed_dev_parts[part].append(replica) # Now we gather partitions that are "at risk" because they aren't # currently sufficient spread out across the cluster. spread_out_parts = defaultdict(list) max_allowed_replicas = self._build_max_replicas_by_tier() for part in xrange(self.parts): # Only move one replica at a time if possible. if part in removed_dev_parts: continue # First, add up the count of replicas at each tier for each # partition. # replicas_at_tier was a "lambda: 0" defaultdict, but profiling # revealed the lambda invocation as a significant cost. replicas_at_tier = {} for dev in self._devs_for_part(part): if dev['id'] not in tfd: tfd[dev['id']] = tiers_for_dev(dev) for tier in tfd[dev['id']]: if tier not in replicas_at_tier: replicas_at_tier[tier] = 1 else: replicas_at_tier[tier] += 1 # Now, look for partitions not yet spread out enough and not # recently moved. for replica in self._replicas_for_part(part): dev = self.devs[self._replica2part2dev[replica][part]] removed_replica = False if dev['id'] not in tfd: tfd[dev['id']] = tiers_for_dev(dev) for tier in tfd[dev['id']]: rep_at_tier = 0 if tier in replicas_at_tier: rep_at_tier = replicas_at_tier[tier] if (rep_at_tier > max_allowed_replicas[tier] and self._last_part_moves[part] >= self.min_part_hours): self._last_part_moves[part] = 0 spread_out_parts[part].append(replica) dev['parts_wanted'] += 1 dev['parts'] -= 1 removed_replica = True break if removed_replica: if dev['id'] not in tfd: tfd[dev['id']] = tiers_for_dev(dev) for tier in tfd[dev['id']]: replicas_at_tier[tier] -= 1 # Last, we gather partitions from devices that are "overweight" because # they have more partitions than their parts_wanted. reassign_parts = defaultdict(list) # We randomly pick a new starting point in the "circular" ring of # partitions to try to get a better rebalance when called multiple # times. start = self._last_part_gather_start / 4 start += random.randint(0, self.parts / 2) # GRAH PEP8!!! self._last_part_gather_start = start for replica, part2dev in enumerate(self._replica2part2dev): # If we've got a partial replica, start may be out of # range. Scale it down so that we get a similar movement # pattern (but scaled down) on sequential runs. this_start = int(float(start) * len(part2dev) / self.parts) for part in itertools.chain(xrange(this_start, len(part2dev)), xrange(0, this_start)): if self._last_part_moves[part] < self.min_part_hours: continue if part in removed_dev_parts or part in spread_out_parts: continue dev = self.devs[part2dev[part]] if dev['parts_wanted'] < 0: self._last_part_moves[part] = 0 dev['parts_wanted'] += 1 dev['parts'] -= 1 reassign_parts[part].append(replica) reassign_parts.update(spread_out_parts) reassign_parts.update(removed_dev_parts) reassign_parts_list = list(reassign_parts.iteritems()) # We shuffle the partitions to reassign so we get a more even # distribution later. There has been discussion of trying to distribute # partitions more "regularly" because that would actually reduce risk # but 1) it is really difficult to do this with uneven clusters and 2) # it would concentrate load during failure recovery scenarios # (increasing risk). The "right" answer has yet to be debated to # conclusion, but working code wins for now. random.shuffle(reassign_parts_list) return reassign_parts_list def _reassign_parts(self, reassign_parts): """ For an existing ring data set, partitions are reassigned similarly to the initial assignment. The devices are ordered by how many partitions they still want and kept in that order throughout the process. The gathered partitions are iterated through, assigning them to devices according to the "most wanted" while keeping the replicas as "far apart" as possible. Two different regions are considered the farthest-apart things, followed by zones, then different ip/port pairs within a zone; the least-far-apart things are different devices with the same ip/port pair in the same zone. If you want more replicas than devices, you won't get all your replicas. :param reassign_parts: An iterable of (part, replicas_to_replace) pairs. replicas_to_replace is an iterable of the replica (an int) to replace for that partition. replicas_to_replace may be shared for multiple partitions, so be sure you do not modify it. """ for dev in self._iter_devs(): dev['sort_key'] = self._sort_key_for(dev) dev['tiers'] = tiers_for_dev(dev) available_devs = \ sorted((d for d in self._iter_devs() if d['weight']), key=lambda x: x['sort_key']) tier2devs = defaultdict(list) tier2sort_key = defaultdict(tuple) tier2dev_sort_key = defaultdict(list) max_tier_depth = 0 for dev in available_devs: for tier in dev['tiers']: tier2devs[tier].append(dev) # <-- starts out sorted! tier2dev_sort_key[tier].append(dev['sort_key']) tier2sort_key[tier] = dev['sort_key'] if len(tier) > max_tier_depth: max_tier_depth = len(tier) tier2children_sets = build_tier_tree(available_devs) tier2children = defaultdict(list) tier2children_sort_key = {} tiers_list = [()] depth = 1 while depth <= max_tier_depth: new_tiers_list = [] for tier in tiers_list: child_tiers = list(tier2children_sets[tier]) child_tiers.sort(key=tier2sort_key.__getitem__) tier2children[tier] = child_tiers tier2children_sort_key[tier] = map( tier2sort_key.__getitem__, child_tiers) new_tiers_list.extend(child_tiers) tiers_list = new_tiers_list depth += 1 for part, replace_replicas in reassign_parts: # Gather up what other tiers (regions, zones, ip/ports, and # devices) the replicas not-to-be-moved are in for this part. other_replicas = defaultdict(int) unique_tiers_by_tier_len = defaultdict(set) for replica in self._replicas_for_part(part): if replica not in replace_replicas: dev = self.devs[self._replica2part2dev[replica][part]] for tier in dev['tiers']: other_replicas[tier] += 1 unique_tiers_by_tier_len[len(tier)].add(tier) for replica in replace_replicas: tier = () depth = 1 while depth <= max_tier_depth: # Order the tiers by how many replicas of this # partition they already have. Then, of the ones # with the smallest number of replicas, pick the # tier with the hungriest drive and then continue # searching in that subtree. # # There are other strategies we could use here, # such as hungriest-tier (i.e. biggest # sum-of-parts-wanted) or picking one at random. # However, hungriest-drive is what was used here # before, and it worked pretty well in practice. # # Note that this allocator will balance things as # evenly as possible at each level of the device # layout. If your layout is extremely unbalanced, # this may produce poor results. # # This used to be a cute, recursive function, but it's been # unrolled for performance. # We sort the tiers here so that, when we look for a tier # with the lowest number of replicas, the first one we # find is the one with the hungriest drive (i.e. drive # with the largest sort_key value). This lets us # short-circuit the search while still ensuring we get the # right tier. candidates_with_replicas = \ unique_tiers_by_tier_len[len(tier) + 1] # Find a tier with the minimal replica count and the # hungriest drive among all the tiers with the minimal # replica count. if len(tier2children[tier]) > \ len(candidates_with_replicas): # There exists at least one tier with 0 other replicas tier = max((t for t in tier2children[tier] if other_replicas[t] == 0), key=tier2sort_key.__getitem__) else: tier = max(tier2children[tier], key=lambda t: (-other_replicas[t], tier2sort_key[t])) depth += 1 dev = tier2devs[tier][-1] dev['parts_wanted'] -= 1 dev['parts'] += 1 old_sort_key = dev['sort_key'] new_sort_key = dev['sort_key'] = self._sort_key_for(dev) for tier in dev['tiers']: other_replicas[tier] += 1 unique_tiers_by_tier_len[len(tier)].add(tier) index = bisect.bisect_left(tier2dev_sort_key[tier], old_sort_key) tier2devs[tier].pop(index) tier2dev_sort_key[tier].pop(index) new_index = bisect.bisect_left(tier2dev_sort_key[tier], new_sort_key) tier2devs[tier].insert(new_index, dev) tier2dev_sort_key[tier].insert(new_index, new_sort_key) new_last_sort_key = tier2dev_sort_key[tier][-1] tier2sort_key[tier] = new_last_sort_key # Now jiggle tier2children values to keep them sorted parent_tier = tier[0:-1] index = bisect.bisect_left( tier2children_sort_key[parent_tier], old_sort_key) popped = tier2children[parent_tier].pop(index) tier2children_sort_key[parent_tier].pop(index) new_index = bisect.bisect_left( tier2children_sort_key[parent_tier], new_last_sort_key) tier2children[parent_tier].insert(new_index, popped) tier2children_sort_key[parent_tier].insert( new_index, new_last_sort_key) self._replica2part2dev[replica][part] = dev['id'] # Just to save memory and keep from accidental reuse. for dev in self._iter_devs(): del dev['sort_key'] del dev['tiers'] def _sort_key_for(self, dev): return (dev['parts_wanted'], random.randint(0, 0xFFFF), dev['id']) def _build_max_replicas_by_tier(self): """ Returns a dict of (tier: replica_count) for all tiers in the ring. There will always be a () entry as the root of the structure, whose replica_count will equal the ring's replica_count. Then there will be (dev_id,) entries for each device, indicating the maximum number of replicas the device might have for any given partition. Anything greater than 1 indicates a partition at serious risk, as the data on that partition will not be stored distinctly at the ring's replica_count. Next there will be (dev_id, ip_port) entries for each device, indicating the maximum number of replicas the device shares with other devices on the same ip_port for any given partition. Anything greater than 1 indicates a partition at elevated risk, as if that ip_port were to fail multiple replicas of that partition would be unreachable. Last there will be (dev_id, ip_port, zone) entries for each device, indicating the maximum number of replicas the device shares with other devices within the same zone for any given partition. Anything greater than 1 indicates a partition at slightly elevated risk, as if that zone were to fail multiple replicas of that partition would be unreachable. Example return dict for the common SAIO setup:: {(): 3, (1,): 1.0, (1, '127.0.0.1:6010'): 1.0, (1, '127.0.0.1:6010', 0): 1.0, (2,): 1.0, (2, '127.0.0.1:6020'): 1.0, (2, '127.0.0.1:6020', 1): 1.0, (3,): 1.0, (3, '127.0.0.1:6030'): 1.0, (3, '127.0.0.1:6030', 2): 1.0, (4,): 1.0, (4, '127.0.0.1:6040'): 1.0, (4, '127.0.0.1:6040', 3): 1.0} """ # Used by walk_tree to know what entries to create for each recursive # call. tier2children = build_tier_tree(self._iter_devs()) def walk_tree(tier, replica_count): mr = {tier: replica_count} if tier in tier2children: subtiers = tier2children[tier] for subtier in subtiers: submax = math.ceil(float(replica_count) / len(subtiers)) mr.update(walk_tree(subtier, submax)) return mr return walk_tree((), self.replicas) def _devs_for_part(self, part): """ Returns a list of devices for a specified partition. Deliberately includes duplicates. """ if self._replica2part2dev is None: return [] return [self.devs[part2dev[part]] for part2dev in self._replica2part2dev if part < len(part2dev)] def _replicas_for_part(self, part): """ Returns a list of replicas for a specified partition. These can be used as indices into self._replica2part2dev without worrying about IndexErrors. """ return [replica for replica, part2dev in enumerate(self._replica2part2dev) if part < len(part2dev)] def _each_part_replica(self): """ Generator yielding every (partition, replica) pair in the ring. """ for replica, part2dev in enumerate(self._replica2part2dev): for part in xrange(len(part2dev)): yield (part, replica) @classmethod def load(cls, builder_file, open=open): """ Obtain RingBuilder instance of the provided builder file :param builder_file: path to builder file to load :return: RingBuilder instance """ builder = pickle.load(open(builder_file, 'rb')) if not hasattr(builder, 'devs'): builder_dict = builder builder = RingBuilder(1, 1, 1) builder.copy_from(builder_dict) for dev in builder.devs: #really old rings didn't have meta keys if dev and 'meta' not in dev: dev['meta'] = '' # NOTE(akscram): An old ring builder file don't contain # replication parameters. if dev: if 'ip' in dev: dev.setdefault('replication_ip', dev['ip']) if 'port' in dev: dev.setdefault('replication_port', dev['port']) return builder def save(self, builder_file): """Serialize this RingBuilder instance to disk. :param builder_file: path to builder file to save """ with open(builder_file, 'wb') as f: pickle.dump(self.to_dict(), f, protocol=2) def search_devs(self, search_values): """Search devices by parameters. :param search_values: a dictionary with search values to filter devices, supported parameters are id, region, zone, ip, port, replication_ip, replication_port, device, weight, meta :returns: list of device dicts """ matched_devs = [] for dev in self.devs: if not dev: continue matched = True for key in ('id', 'region', 'zone', 'ip', 'port', 'replication_ip', 'replication_port', 'device', 'weight', 'meta'): if key in search_values: value = search_values.get(key) if value is not None: if key == 'meta': if value not in dev.get(key): matched = False elif dev.get(key) != value: matched = False if matched: matched_devs.append(dev) return matched_devs swift-1.13.1/swift/common/ring/__init__.py0000664000175400017540000000024212323703611021557 0ustar jenkinsjenkins00000000000000from swift.common.ring.ring import RingData, Ring from swift.common.ring.builder import RingBuilder __all__ = [ 'RingData', 'Ring', 'RingBuilder', ] swift-1.13.1/swift/common/direct_client.py0000664000175400017540000004604412323703611021703 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Internal client library for making calls directly to the servers rather than through the proxy. """ import os import socket from httplib import HTTPException from time import time from eventlet import sleep, Timeout from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ClientException from swift.common.utils import normalize_timestamp, FileLikeIter from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error from swift.common.swob import HeaderKeyDict from swift.common.utils import quote try: import simplejson as json except ImportError: import json def _get_direct_account_container(path, stype, node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """Base class for get direct account and container. Do not use directly use the get_direct_account or get_direct_container instead. """ qs = 'format=json' if marker: qs += '&marker=%s' % quote(marker) if limit: qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) if delimiter: qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, query_string=qs, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise ClientException( '%s server %s:%s direct GET %s gave stats %s' % (stype, node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value if resp.status == HTTP_NO_CONTENT: resp.read() return resp_headers, [] return resp_headers, json.loads(resp.read()) def gen_headers(hdrs_in=None, add_ts=False): hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() if add_ts: hdrs_out['X-Timestamp'] = normalize_timestamp(time()) hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() return hdrs_out def direct_get_account(node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """ Get listings directly from the account server. :param node: node dictionary from the ring :param part: partition the account is on :param account: account name :param marker: marker query :param limit: query limit :param prefix: prefix query :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. """ path = '/' + account return _get_direct_account_container(path, "Account", node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15) def direct_head_container(node, part, account, container, conn_timeout=5, response_timeout=15): """ Request container information directly from the container server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a dict containing the response's headers (all header names will be lowercase) """ path = '/%s/%s' % (account, container) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'HEAD', path, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Container server %s:%s direct HEAD %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers def direct_get_container(node, part, account, container, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): """ Get container listings directly from the container server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param marker: marker query :param limit: query limit :param prefix: prefix query :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. """ path = '/%s/%s' % (account, container) return _get_direct_account_container(path, "Container", node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15) def direct_delete_container(node, part, account, container, conn_timeout=5, response_timeout=15, headers=None): if headers is None: headers = {} path = '/%s/%s' % (account, container) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'DELETE', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Container server %s:%s direct DELETE %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) def direct_head_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15): """ Request object information directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: a dict containing the response's headers (all header names will be lowercase) """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'HEAD', path, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct HEAD %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers def direct_get_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, resp_chunk_size=None, headers=None): """ Get object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param resp_chunk_size: if defined, chunk size of data to read. :param headers: dict to be passed into HTTPConnection headers :returns: a tuple of (response headers, the object's contents) The response headers will be a dict and all header names will be lowercase. """ if headers is None: headers = {} path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'GET', path, headers=gen_headers(headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) if resp_chunk_size: def _object_body(): buf = resp.read(resp_chunk_size) while buf: yield buf buf = resp.read(resp_chunk_size) object_body = _object_body() else: object_body = resp.read() resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value return resp_headers, object_body def direct_put_object(node, part, account, container, name, contents, content_length=None, etag=None, content_type=None, headers=None, conn_timeout=5, response_timeout=15, chunk_size=65535): """ Put object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param contents: an iterable or string to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param content_type: value to send as content-type header :param headers: additional headers to include in the request :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param chunk_size: if defined, chunk size of data to send. :returns: etag from the server response """ path = '/%s/%s/%s' % (account, container, name) if headers is None: headers = {} if etag: headers['ETag'] = etag.strip('"') if content_length is not None: headers['Content-Length'] = str(content_length) else: for n, v in headers.iteritems(): if n.lower() == 'content-length': content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type else: headers['Content-Type'] = 'application/octet-stream' if not contents: headers['Content-Length'] = '0' if isinstance(contents, basestring): contents = [contents] #Incase the caller want to insert an object with specific age add_ts = 'X-Timestamp' not in headers if content_length is None: headers['Transfer-Encoding'] = 'chunked' with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'PUT', path, headers=gen_headers(headers, add_ts)) contents_f = FileLikeIter(contents) if content_length is None: chunk = contents_f.read(chunk_size) while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = contents_f.read(chunk_size) conn.send('0\r\n\r\n') else: left = content_length while left > 0: size = chunk_size if size > left: size = left chunk = contents_f.read(size) if not chunk: break conn.send(chunk) left -= len(chunk) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct PUT %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) return resp.getheader('etag').strip('"') def direct_post_object(node, part, account, container, name, headers, conn_timeout=5, response_timeout=15): """ Direct update to object metadata on object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param name: object name :param headers: headers to store as metadata :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :raises ClientException: HTTP POST request failed """ path = '/%s/%s/%s' % (account, container, name) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'POST', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct POST %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) def direct_delete_object(node, part, account, container, obj, conn_timeout=5, response_timeout=15, headers=None): """ Delete object directly from the object server. :param node: node dictionary from the ring :param part: partition the container is on :param account: account name :param container: container name :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :returns: response from server """ if headers is None: headers = {} path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, 'DELETE', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() if not is_success(resp.status): raise ClientException( 'Object server %s:%s direct DELETE %s gave status %s' % (node['ip'], node['port'], repr('/%s/%s%s' % (node['device'], part, path)), resp.status), http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) def retry(func, *args, **kwargs): """ Helper function to retry a given function a number of times. :param func: callable to be called :param retries: number of retries :param error_log: logger for errors :param args: arguments to send to func :param kwargs: keyward arguments to send to func (if retries or error_log are sent, they will be deleted from kwargs before sending on to func) :returns: restult of func """ retries = 5 if 'retries' in kwargs: retries = kwargs['retries'] del kwargs['retries'] error_log = None if 'error_log' in kwargs: error_log = kwargs['error_log'] del kwargs['error_log'] attempts = 0 backoff = 1 while attempts <= retries: attempts += 1 try: return attempts, func(*args, **kwargs) except (socket.error, HTTPException, Timeout) as err: if error_log: error_log(err) if attempts > retries: raise except ClientException as err: if error_log: error_log(err) if attempts > retries or not is_server_error(err.http_status) or \ err.http_status == HTTP_INSUFFICIENT_STORAGE: raise sleep(backoff) backoff *= 2 # Shouldn't actually get down here, but just in case. if args and 'ip' in args[0]: raise ClientException('Raise too many retries', http_host=args[ 0]['ip'], http_port=args[0]['port'], http_device=args[0]['device']) else: raise ClientException('Raise too many retries') swift-1.13.1/swift/common/http.py0000664000175400017540000001077412323703611020053 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. def is_informational(status): """ Check if HTTP status code is informational. :param status: http status code :returns: True if status is successful, else False """ return 100 <= status <= 199 def is_success(status): """ Check if HTTP status code is successful. :param status: http status code :returns: True if status is successful, else False """ return 200 <= status <= 299 def is_redirection(status): """ Check if HTTP status code is redirection. :param status: http status code :returns: True if status is redirection, else False """ return 300 <= status <= 399 def is_client_error(status): """ Check if HTTP status code is client error. :param status: http status code :returns: True if status is client error, else False """ return 400 <= status <= 499 def is_server_error(status): """ Check if HTTP status code is server error. :param status: http status code :returns: True if status is server error, else False """ return 500 <= status <= 599 # List of HTTP status codes ############################################################################### ## 1xx Informational ############################################################################### HTTP_CONTINUE = 100 HTTP_SWITCHING_PROTOCOLS = 101 HTTP_PROCESSING = 102 # WebDAV HTTP_CHECKPOINT = 103 HTTP_REQUEST_URI_TOO_LONG = 122 ############################################################################### ## 2xx Success ############################################################################### HTTP_OK = 200 HTTP_CREATED = 201 HTTP_ACCEPTED = 202 HTTP_NON_AUTHORITATIVE_INFORMATION = 203 HTTP_NO_CONTENT = 204 HTTP_RESET_CONTENT = 205 HTTP_PARTIAL_CONTENT = 206 HTTP_MULTI_STATUS = 207 # WebDAV HTTP_IM_USED = 226 ############################################################################### ## 3xx Redirection ############################################################################### HTTP_MULTIPLE_CHOICES = 300 HTTP_MOVED_PERMANENTLY = 301 HTTP_FOUND = 302 HTTP_SEE_OTHER = 303 HTTP_NOT_MODIFIED = 304 HTTP_USE_PROXY = 305 HTTP_SWITCH_PROXY = 306 HTTP_TEMPORARY_REDIRECT = 307 HTTP_RESUME_INCOMPLETE = 308 ############################################################################### ## 4xx Client Error ############################################################################### HTTP_BAD_REQUEST = 400 HTTP_UNAUTHORIZED = 401 HTTP_PAYMENT_REQUIRED = 402 HTTP_FORBIDDEN = 403 HTTP_NOT_FOUND = 404 HTTP_METHOD_NOT_ALLOWED = 405 HTTP_NOT_ACCEPTABLE = 406 HTTP_PROXY_AUTHENTICATION_REQUIRED = 407 HTTP_REQUEST_TIMEOUT = 408 HTTP_CONFLICT = 409 HTTP_GONE = 410 HTTP_LENGTH_REQUIRED = 411 HTTP_PRECONDITION_FAILED = 412 HTTP_REQUEST_ENTITY_TOO_LARGE = 413 HTTP_REQUEST_URI_TOO_LONG = 414 HTTP_UNSUPPORTED_MEDIA_TYPE = 415 HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416 HTTP_EXPECTATION_FAILED = 417 HTTP_IM_A_TEAPOT = 418 HTTP_UNPROCESSABLE_ENTITY = 422 # WebDAV HTTP_LOCKED = 423 # WebDAV HTTP_FAILED_DEPENDENCY = 424 # WebDAV HTTP_UNORDERED_COLLECTION = 425 HTTP_UPGRADE_REQUIED = 426 HTTP_PRECONDITION_REQUIRED = 428 HTTP_TOO_MANY_REQUESTS = 429 HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE = 431 HTTP_NO_RESPONSE = 444 HTTP_RETRY_WITH = 449 HTTP_BLOCKED_BY_WINDOWS_PARENTAL_CONTROLS = 450 HTTP_CLIENT_CLOSED_REQUEST = 499 ############################################################################### ## 5xx Server Error ############################################################################### HTTP_INTERNAL_SERVER_ERROR = 500 HTTP_NOT_IMPLEMENTED = 501 HTTP_BAD_GATEWAY = 502 HTTP_SERVICE_UNAVAILABLE = 503 HTTP_GATEWAY_TIMEOUT = 504 HTTP_VERSION_NOT_SUPPORTED = 505 HTTP_VARIANT_ALSO_NEGOTIATES = 506 HTTP_INSUFFICIENT_STORAGE = 507 # WebDAV HTTP_BANDWIDTH_LIMIT_EXCEEDED = 509 HTTP_NOT_EXTENDED = 510 HTTP_NETWORK_AUTHENTICATION_REQUIRED = 511 HTTP_NETWORK_READ_TIMEOUT_ERROR = 598 # not used in RFC HTTP_NETWORK_CONNECT_TIMEOUT_ERROR = 599 # not used in RFC swift-1.13.1/swift/common/memcached.py0000664000175400017540000005254212323703611021001 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Why our own memcache client? By Michael Barton python-memcached doesn't use consistent hashing, so adding or removing a memcache server from the pool invalidates a huge percentage of cached items. If you keep a pool of python-memcached client objects, each client object has its own connection to every memcached server, only one of which is ever in use. So you wind up with n * m open sockets and almost all of them idle. This client effectively has a pool for each server, so the number of backend connections is hopefully greatly reduced. python-memcache uses pickle to store things, and there was already a huge stink about Swift using pickles in memcache (http://osvdb.org/show/osvdb/86581). That seemed sort of unfair, since nova and keystone and everyone else use pickles for memcache too, but it's hidden behind a "standard" library. But changing would be a security regression at this point. Also, pylibmc wouldn't work for us because it needs to use python sockets in order to play nice with eventlet. Lucid comes with memcached: v1.4.2. Protocol documentation for that version is at: http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt """ import cPickle as pickle import logging import time from bisect import bisect from swift import gettext_ as _ from hashlib import md5 from distutils.version import StrictVersion from eventlet.green import socket from eventlet.pools import Pool from eventlet import Timeout, __version__ as eventlet_version from swift.common.utils import json DEFAULT_MEMCACHED_PORT = 11211 CONN_TIMEOUT = 0.3 POOL_TIMEOUT = 1.0 # WAG IO_TIMEOUT = 2.0 PICKLE_FLAG = 1 JSON_FLAG = 2 NODE_WEIGHT = 50 PICKLE_PROTOCOL = 2 TRY_COUNT = 3 # if ERROR_LIMIT_COUNT errors occur in ERROR_LIMIT_TIME seconds, the server # will be considered failed for ERROR_LIMIT_DURATION seconds. ERROR_LIMIT_COUNT = 10 ERROR_LIMIT_TIME = 60 ERROR_LIMIT_DURATION = 60 def md5hash(key): return md5(key).hexdigest() def sanitize_timeout(timeout): """ Sanitize a timeout value to use an absolute expiration time if the delta is greater than 30 days (in seconds). Note that the memcached server translates negative values to mean a delta of 30 days in seconds (and 1 additional second), client beware. """ if timeout > (30 * 24 * 60 * 60): timeout += time.time() return timeout class MemcacheConnectionError(Exception): pass class MemcachePoolTimeout(Timeout): pass class MemcacheConnPool(Pool): """Connection pool for Memcache Connections""" def __init__(self, server, size, connect_timeout): Pool.__init__(self, max_size=size) self.server = server self._connect_timeout = connect_timeout self._parent_class_getter = super(MemcacheConnPool, self).get try: # call the patched .get() if eventlet is older than 0.9.17 if StrictVersion(eventlet_version) < StrictVersion('0.9.17'): self._parent_class_getter = self._upstream_fixed_get except ValueError: # "invalid" version number or otherwise error parsing version pass def create(self): if ':' in self.server: host, port = self.server.split(':') else: host = self.server port = DEFAULT_MEMCACHED_PORT sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) with Timeout(self._connect_timeout): sock.connect((host, int(port))) return (sock.makefile(), sock) def get(self): fp, sock = self._parent_class_getter() if fp is None: # An error happened previously, so we need a new connection fp, sock = self.create() return fp, sock # The following method is from eventlet post 0.9.16. This version # properly keeps track of pool size accounting, and therefore doesn't # let the pool grow without bound. This patched version is the result # of commit f5e5b2bda7b442f0262ee1084deefcc5a1cc0694 in eventlet and is # documented at https://bitbucket.org/eventlet/eventlet/issue/91 def _upstream_fixed_get(self): """Return an item from the pool, when one is available. This may cause the calling greenthread to block. """ if self.free_items: return self.free_items.popleft() self.current_size += 1 if self.current_size <= self.max_size: try: created = self.create() except: # noqa self.current_size -= 1 raise return created self.current_size -= 1 # did not create return self.channel.get() class MemcacheRing(object): """ Simple, consistent-hashed memcache client. """ def __init__(self, servers, connect_timeout=CONN_TIMEOUT, io_timeout=IO_TIMEOUT, pool_timeout=POOL_TIMEOUT, tries=TRY_COUNT, allow_pickle=False, allow_unpickle=False, max_conns=2): self._ring = {} self._errors = dict(((serv, []) for serv in servers)) self._error_limited = dict(((serv, 0) for serv in servers)) for server in sorted(servers): for i in xrange(NODE_WEIGHT): self._ring[md5hash('%s-%s' % (server, i))] = server self._tries = tries if tries <= len(servers) else len(servers) self._sorted = sorted(self._ring) self._client_cache = dict(((server, MemcacheConnPool(server, max_conns, connect_timeout)) for server in servers)) self._connect_timeout = connect_timeout self._io_timeout = io_timeout self._pool_timeout = pool_timeout self._allow_pickle = allow_pickle self._allow_unpickle = allow_unpickle or allow_pickle def _exception_occurred(self, server, e, action='talking', sock=None, fp=None, got_connection=True): if isinstance(e, Timeout): logging.error(_("Timeout %(action)s to memcached: %(server)s"), {'action': action, 'server': server}) else: logging.exception(_("Error %(action)s to memcached: %(server)s"), {'action': action, 'server': server}) try: if fp: fp.close() del fp except Exception: pass try: if sock: sock.close() del sock except Exception: pass if got_connection: # We need to return something to the pool # A new connection will be created the next time it is retreived self._return_conn(server, None, None) now = time.time() self._errors[server].append(time.time()) if len(self._errors[server]) > ERROR_LIMIT_COUNT: self._errors[server] = [err for err in self._errors[server] if err > now - ERROR_LIMIT_TIME] if len(self._errors[server]) > ERROR_LIMIT_COUNT: self._error_limited[server] = now + ERROR_LIMIT_DURATION logging.error(_('Error limiting server %s'), server) def _get_conns(self, key): """ Retrieves a server conn from the pool, or connects a new one. Chooses the server based on a consistent hash of "key". """ pos = bisect(self._sorted, key) served = [] while len(served) < self._tries: pos = (pos + 1) % len(self._sorted) server = self._ring[self._sorted[pos]] if server in served: continue served.append(server) if self._error_limited[server] > time.time(): continue sock = None try: with MemcachePoolTimeout(self._pool_timeout): fp, sock = self._client_cache[server].get() yield server, fp, sock except MemcachePoolTimeout as e: self._exception_occurred( server, e, action='getting a connection', got_connection=False) except (Exception, Timeout) as e: # Typically a Timeout exception caught here is the one raised # by the create() method of this server's MemcacheConnPool # object. self._exception_occurred( server, e, action='connecting', sock=sock) def _return_conn(self, server, fp, sock): """Returns a server connection to the pool.""" self._client_cache[server].put((fp, sock)) def set(self, key, value, serialize=True, timeout=0, time=0, min_compress_len=0): """ Set a key/value pair in memcache :param key: key :param value: value :param serialize: if True, value is serialized with JSON before sending to memcache, or with pickle if configured to use pickle instead of JSON (to avoid cache poisoning) :param timeout: ttl in memcache, this parameter is now deprecated. It will be removed in next release of OpenStack, use time parameter instead in the future :time: equivalent to timeout, this parameter is added to keep the signature compatible with python-memcached interface. This implementation will take this value and sign it to the parameter timeout :min_compress_len: minimum compress length, this parameter was added to keep the signature compatible with python-memcached interface. This implementation ignores it. """ key = md5hash(key) if timeout: logging.warn("parameter timeout has been deprecated, use time") timeout = sanitize_timeout(time or timeout) flags = 0 if serialize and self._allow_pickle: value = pickle.dumps(value, PICKLE_PROTOCOL) flags |= PICKLE_FLAG elif serialize: value = json.dumps(value) flags |= JSON_FLAG for (server, fp, sock) in self._get_conns(key): try: with Timeout(self._io_timeout): sock.sendall('set %s %d %d %s\r\n%s\r\n' % (key, flags, timeout, len(value), value)) # Wait for the set to complete fp.readline() self._return_conn(server, fp, sock) return except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) def get(self, key): """ Gets the object specified by key. It will also unserialize the object before returning if it is serialized in memcache with JSON, or if it is pickled and unpickling is allowed. :param key: key :returns: value of the key in memcache """ key = md5hash(key) value = None for (server, fp, sock) in self._get_conns(key): try: with Timeout(self._io_timeout): sock.sendall('get %s\r\n' % key) line = fp.readline().strip().split() while line[0].upper() != 'END': if line[0].upper() == 'VALUE' and line[1] == key: size = int(line[3]) value = fp.read(size) if int(line[2]) & PICKLE_FLAG: if self._allow_unpickle: value = pickle.loads(value) else: value = None elif int(line[2]) & JSON_FLAG: value = json.loads(value) fp.readline() line = fp.readline().strip().split() self._return_conn(server, fp, sock) return value except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) def incr(self, key, delta=1, time=0, timeout=0): """ Increments a key which has a numeric value by delta. If the key can't be found, it's added as delta or 0 if delta < 0. If passed a negative number, will use memcached's decr. Returns the int stored in memcached Note: The data memcached stores as the result of incr/decr is an unsigned int. decr's that result in a number below 0 are stored as 0. :param key: key :param delta: amount to add to the value of key (or set as the value if the key is not found) will be cast to an int :param time: the time to live. This parameter deprecates parameter timeout. The addition of this parameter is to make the interface consistent with set and set_multi methods :param timeout: ttl in memcache, deprecated, will be removed in future OpenStack releases :returns: result of incrementing :raises MemcacheConnectionError: """ if timeout: logging.warn("parameter timeout has been deprecated, use time") key = md5hash(key) command = 'incr' if delta < 0: command = 'decr' delta = str(abs(int(delta))) timeout = sanitize_timeout(time or timeout) for (server, fp, sock) in self._get_conns(key): try: with Timeout(self._io_timeout): sock.sendall('%s %s %s\r\n' % (command, key, delta)) line = fp.readline().strip().split() if line[0].upper() == 'NOT_FOUND': add_val = delta if command == 'decr': add_val = '0' sock.sendall('add %s %d %d %s\r\n%s\r\n' % (key, 0, timeout, len(add_val), add_val)) line = fp.readline().strip().split() if line[0].upper() == 'NOT_STORED': sock.sendall('%s %s %s\r\n' % (command, key, delta)) line = fp.readline().strip().split() ret = int(line[0].strip()) else: ret = int(add_val) else: ret = int(line[0].strip()) self._return_conn(server, fp, sock) return ret except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) raise MemcacheConnectionError("No Memcached connections succeeded.") def decr(self, key, delta=1, time=0, timeout=0): """ Decrements a key which has a numeric value by delta. Calls incr with -delta. :param key: key :param delta: amount to subtract to the value of key (or set the value to 0 if the key is not found) will be cast to an int :param time: the time to live. This parameter depcates parameter timeout. The addition of this parameter is to make the interface consistent with set and set_multi methods :param timeout: ttl in memcache, deprecated, will be removed in future OpenStack releases :returns: result of decrementing :raises MemcacheConnectionError: """ if timeout: logging.warn("parameter timeout has been deprecated, use time") return self.incr(key, delta=-delta, time=(time or timeout)) def delete(self, key): """ Deletes a key/value pair from memcache. :param key: key to be deleted """ key = md5hash(key) for (server, fp, sock) in self._get_conns(key): try: with Timeout(self._io_timeout): sock.sendall('delete %s\r\n' % key) # Wait for the delete to complete fp.readline() self._return_conn(server, fp, sock) return except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) def set_multi(self, mapping, server_key, serialize=True, timeout=0, time=0, min_compress_len=0): """ Sets multiple key/value pairs in memcache. :param mapping: dictonary of keys and values to be set in memcache :param servery_key: key to use in determining which server in the ring is used :param serialize: if True, value is serialized with JSON before sending to memcache, or with pickle if configured to use pickle instead of JSON (to avoid cache poisoning) :param timeout: ttl for memcache. This parameter is now deprecated, it will be removed in next release of OpenStack, use time parameter instead in the future :time: equalvent to timeout, this parameter is added to keep the signature compatible with python-memcached interface. This implementation will take this value and sign it to parameter timeout :min_compress_len: minimum compress length, this parameter was added to keep the signature compatible with python-memcached interface. This implementation ignores it """ if timeout: logging.warn("parameter timeout has been deprecated, use time") server_key = md5hash(server_key) timeout = sanitize_timeout(time or timeout) msg = '' for key, value in mapping.iteritems(): key = md5hash(key) flags = 0 if serialize and self._allow_pickle: value = pickle.dumps(value, PICKLE_PROTOCOL) flags |= PICKLE_FLAG elif serialize: value = json.dumps(value) flags |= JSON_FLAG msg += ('set %s %d %d %s\r\n%s\r\n' % (key, flags, timeout, len(value), value)) for (server, fp, sock) in self._get_conns(server_key): try: with Timeout(self._io_timeout): sock.sendall(msg) # Wait for the set to complete for _ in range(len(mapping)): fp.readline() self._return_conn(server, fp, sock) return except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) def get_multi(self, keys, server_key): """ Gets multiple values from memcache for the given keys. :param keys: keys for values to be retrieved from memcache :param servery_key: key to use in determining which server in the ring is used :returns: list of values """ server_key = md5hash(server_key) keys = [md5hash(key) for key in keys] for (server, fp, sock) in self._get_conns(server_key): try: with Timeout(self._io_timeout): sock.sendall('get %s\r\n' % ' '.join(keys)) line = fp.readline().strip().split() responses = {} while line[0].upper() != 'END': if line[0].upper() == 'VALUE': size = int(line[3]) value = fp.read(size) if int(line[2]) & PICKLE_FLAG: if self._allow_unpickle: value = pickle.loads(value) else: value = None elif int(line[2]) & JSON_FLAG: value = json.loads(value) responses[line[1]] = value fp.readline() line = fp.readline().strip().split() values = [] for key in keys: if key in responses: values.append(responses[key]) else: values.append(None) self._return_conn(server, fp, sock) return values except (Exception, Timeout) as e: self._exception_occurred(server, e, sock=sock, fp=fp) swift-1.13.1/swift/common/__init__.py0000664000175400017540000000004312323703611020617 0ustar jenkinsjenkins00000000000000"""Code common to all of Swift.""" swift-1.13.1/swift/common/db_replicator.py0000664000175400017540000007304212323703611021702 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import random import math import time import shutil import uuid import errno import re from swift import gettext_ as _ from eventlet import GreenPool, sleep, Timeout from eventlet.green import subprocess import simplejson import swift.common.db from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ unlink_older_than, dump_recon_cache, rsync_ip, ismount from swift.common import ring from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE from swift.common.bufferedhttp import BufferedHTTPConnection from swift.common.exceptions import DriveNotMounted, ConnectionTimeout from swift.common.daemon import Daemon from swift.common.swob import Response, HTTPNotFound, HTTPNoContent, \ HTTPAccepted, HTTPBadRequest DEBUG_TIMINGS_THRESHOLD = 10 def quarantine_db(object_file, server_type): """ In the case that a corrupt file is found, move it to a quarantined area to allow replication to fix it. :param object_file: path to corrupt file :param server_type: type of file that is corrupt ('container' or 'account') """ object_dir = os.path.dirname(object_file) quarantine_dir = os.path.abspath( os.path.join(object_dir, '..', '..', '..', '..', 'quarantined', server_type + 's', os.path.basename(object_dir))) try: renamer(object_dir, quarantine_dir) except OSError as e: if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): raise quarantine_dir = "%s-%s" % (quarantine_dir, uuid.uuid4().hex) renamer(object_dir, quarantine_dir) def roundrobin_datadirs(datadirs): """ Generator to walk the data dirs in a round robin manner, evenly hitting each device on the system, and yielding any .db files found (in their proper places). The partitions within each data dir are walked randomly, however. :param datadirs: a list of (path, node_id) to walk :returns: A generator of (partition, path_to_db_file, node_id) """ def walk_datadir(datadir, node_id): partitions = os.listdir(datadir) random.shuffle(partitions) for partition in partitions: part_dir = os.path.join(datadir, partition) if not os.path.isdir(part_dir): continue suffixes = os.listdir(part_dir) for suffix in suffixes: suff_dir = os.path.join(part_dir, suffix) if not os.path.isdir(suff_dir): continue hashes = os.listdir(suff_dir) for hsh in hashes: hash_dir = os.path.join(suff_dir, hsh) if not os.path.isdir(hash_dir): continue object_file = os.path.join(hash_dir, hsh + '.db') if os.path.exists(object_file): yield (partition, object_file, node_id) its = [walk_datadir(datadir, node_id) for datadir, node_id in datadirs] while its: for it in its: try: yield it.next() except StopIteration: its.remove(it) class ReplConnection(BufferedHTTPConnection): """ Helper to simplify REPLICATEing to a remote server. """ def __init__(self, node, partition, hash_, logger): "" self.logger = logger self.node = node host = "%s:%s" % (node['replication_ip'], node['replication_port']) BufferedHTTPConnection.__init__(self, host) self.path = '/%s/%s/%s' % (node['device'], partition, hash_) def replicate(self, *args): """ Make an HTTP REPLICATE request :param args: list of json-encodable objects :returns: bufferedhttp response object """ try: body = simplejson.dumps(args) self.request('REPLICATE', self.path, body, {'Content-Type': 'application/json'}) response = self.getresponse() response.data = response.read() return response except (Exception, Timeout): self.logger.exception( _('ERROR reading HTTP response from %s'), self.node) return None class Replicator(Daemon): """ Implements the logic for directing db replication. """ def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='replicator') self.root = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.port = int(conf.get('bind_port', self.default_port)) concurrency = int(conf.get('concurrency', 8)) self.cpool = GreenPool(size=concurrency) swift_dir = conf.get('swift_dir', '/etc/swift') self.ring = ring.Ring(swift_dir, ring_name=self.server_type) self.per_diff = int(conf.get('per_diff', 1000)) self.max_diffs = int(conf.get('max_diffs') or 100) self.interval = int(conf.get('interval') or conf.get('run_pause') or 30) self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self._zero_stats() self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.recon_replicator = '%s.recon' % self.server_type self.rcache = os.path.join(self.recon_cache_path, self.recon_replicator) self.extract_device_re = re.compile('%s%s([^%s]+)' % ( self.root, os.path.sep, os.path.sep)) def _zero_stats(self): """Zero out the stats.""" self.stats = {'attempted': 0, 'success': 0, 'failure': 0, 'ts_repl': 0, 'no_change': 0, 'hashmatch': 0, 'rsync': 0, 'diff': 0, 'remove': 0, 'empty': 0, 'remote_merge': 0, 'start': time.time(), 'diff_capped': 0} def _report_stats(self): """Report the current stats to the logs.""" now = time.time() self.logger.info( _('Attempted to replicate %(count)d dbs in %(time).5f seconds ' '(%(rate).5f/s)'), {'count': self.stats['attempted'], 'time': now - self.stats['start'], 'rate': self.stats['attempted'] / (now - self.stats['start'] + 0.0000001)}) self.logger.info(_('Removed %(remove)d dbs') % self.stats) self.logger.info(_('%(success)s successes, %(failure)s failures') % self.stats) dump_recon_cache( {'replication_stats': self.stats, 'replication_time': now - self.stats['start'], 'replication_last': now}, self.rcache, self.logger) self.logger.info(' '.join(['%s:%s' % item for item in self.stats.items() if item[0] in ('no_change', 'hashmatch', 'rsync', 'diff', 'ts_repl', 'empty', 'diff_capped')])) def _rsync_file(self, db_file, remote_file, whole_file=True): """ Sync a single file using rsync. Used by _rsync_db to handle syncing. :param db_file: file to be synced :param remote_file: remote location to sync the DB file to :param whole-file: if True, uses rsync's --whole-file flag :returns: True if the sync was successful, False otherwise """ popen_args = ['rsync', '--quiet', '--no-motd', '--timeout=%s' % int(math.ceil(self.node_timeout)), '--contimeout=%s' % int(math.ceil(self.conn_timeout))] if whole_file: popen_args.append('--whole-file') popen_args.extend([db_file, remote_file]) proc = subprocess.Popen(popen_args) proc.communicate() if proc.returncode != 0: self.logger.error(_('ERROR rsync failed with %(code)s: %(args)s'), {'code': proc.returncode, 'args': popen_args}) return proc.returncode == 0 def _rsync_db(self, broker, device, http, local_id, replicate_method='complete_rsync', replicate_timeout=None): """ Sync a whole db using rsync. :param broker: DB broker object of DB to be synced :param device: device to sync to :param http: ReplConnection object :param local_id: unique ID of the local database replica :param replicate_method: remote operation to perform after rsync :param replicate_timeout: timeout to wait in seconds """ device_ip = rsync_ip(device['replication_ip']) if self.vm_test_mode: remote_file = '%s::%s%s/%s/tmp/%s' % ( device_ip, self.server_type, device['replication_port'], device['device'], local_id) else: remote_file = '%s::%s/%s/tmp/%s' % ( device_ip, self.server_type, device['device'], local_id) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, False): return False with Timeout(replicate_timeout or self.node_timeout): response = http.replicate(replicate_method, local_id) return response and response.status >= 200 and response.status < 300 def _usync_db(self, point, broker, http, remote_id, local_id): """ Sync a db by sending all records since the last sync. :param point: synchronization high water mark between the replicas :param broker: database broker object :param http: ReplConnection object for the remote server :param remote_id: database id for the remote replica :param local_id: database id for the local replica :returns: boolean indicating completion and success """ self.stats['diff'] += 1 self.logger.increment('diffs') self.logger.debug(_('Syncing chunks with %s'), http.host) sync_table = broker.get_syncs() objects = broker.get_items_since(point, self.per_diff) diffs = 0 while len(objects) and diffs < self.max_diffs: diffs += 1 with Timeout(self.node_timeout): response = http.replicate('merge_items', objects, local_id) if not response or response.status >= 300 or response.status < 200: if response: self.logger.error(_('ERROR Bad response %(status)s from ' '%(host)s'), {'status': response.status, 'host': http.host}) return False point = objects[-1]['ROWID'] objects = broker.get_items_since(point, self.per_diff) if objects: self.logger.debug(_( 'Synchronization for %s has fallen more than ' '%s rows behind; moving on and will try again next pass.'), broker, self.max_diffs * self.per_diff) self.stats['diff_capped'] += 1 self.logger.increment('diff_caps') else: with Timeout(self.node_timeout): response = http.replicate('merge_syncs', sync_table) if response and response.status >= 200 and response.status < 300: broker.merge_syncs([{'remote_id': remote_id, 'sync_point': point}], incoming=False) return True return False def _in_sync(self, rinfo, info, broker, local_sync): """ Determine whether or not two replicas of a databases are considered to be in sync. :param rinfo: remote database info :param info: local database info :param broker: database broker object :param local_sync: cached last sync point between replicas :returns: boolean indicating whether or not the replicas are in sync """ if max(rinfo['point'], local_sync) >= info['max_row']: self.stats['no_change'] += 1 self.logger.increment('no_changes') return True if rinfo['hash'] == info['hash']: self.stats['hashmatch'] += 1 self.logger.increment('hashmatches') broker.merge_syncs([{'remote_id': rinfo['id'], 'sync_point': rinfo['point']}], incoming=False) return True def _http_connect(self, node, partition, db_file): """ Make an http_connection using ReplConnection :param node: node dictionary from the ring :param partition: partition partition to send in the url :param db_file: DB file :returns: ReplConnection object """ return ReplConnection(node, partition, os.path.basename(db_file).split('.', 1)[0], self.logger) def _repl_to_node(self, node, broker, partition, info): """ Replicate a database to a node. :param node: node dictionary from the ring to be replicated to :param broker: DB broker for the DB to be replication :param partition: partition on the node to replicate to :param info: DB info as a dictionary of {'max_row', 'hash', 'id', 'created_at', 'put_timestamp', 'delete_timestamp', 'metadata'} :returns: True if successful, False otherwise """ with ConnectionTimeout(self.conn_timeout): http = self._http_connect(node, partition, broker.db_file) if not http: self.logger.error( _('ERROR Unable to connect to remote server: %s'), node) return False with Timeout(self.node_timeout): response = http.replicate( 'sync', info['max_row'], info['hash'], info['id'], info['created_at'], info['put_timestamp'], info['delete_timestamp'], info['metadata']) if not response: return False elif response.status == HTTP_NOT_FOUND: # completely missing, rsync self.stats['rsync'] += 1 self.logger.increment('rsyncs') return self._rsync_db(broker, node, http, info['id']) elif response.status == HTTP_INSUFFICIENT_STORAGE: raise DriveNotMounted() elif response.status >= 200 and response.status < 300: rinfo = simplejson.loads(response.data) local_sync = broker.get_sync(rinfo['id'], incoming=False) if self._in_sync(rinfo, info, broker, local_sync): return True # if the difference in rowids between the two differs by # more than 50%, rsync then do a remote merge. if rinfo['max_row'] / float(info['max_row']) < 0.5: self.stats['remote_merge'] += 1 self.logger.increment('remote_merges') return self._rsync_db(broker, node, http, info['id'], replicate_method='rsync_then_merge', replicate_timeout=(info['count'] / 2000)) # else send diffs over to the remote server return self._usync_db(max(rinfo['point'], local_sync), broker, http, rinfo['id'], info['id']) def _replicate_object(self, partition, object_file, node_id): """ Replicate the db, choosing method based on whether or not it already exists on peers. :param partition: partition to be replicated to :param object_file: DB file name to be replicated :param node_id: node id of the node to be replicated to """ start_time = now = time.time() self.logger.debug(_('Replicating db %s'), object_file) self.stats['attempted'] += 1 self.logger.increment('attempts') shouldbehere = True try: broker = self.brokerclass(object_file, pending_timeout=30) broker.reclaim(now - self.reclaim_age, now - (self.reclaim_age * 2)) info = broker.get_replication_info() full_info = broker.get_info() bpart = self.ring.get_part( full_info['account'], full_info.get('container')) if bpart != int(partition): partition = bpart # Important to set this false here since the later check only # checks if it's on the proper device, not partition. shouldbehere = False name = '/' + quote(full_info['account']) if 'container' in full_info: name += '/' + quote(full_info['container']) self.logger.error( 'Found %s for %s when it should be on partition %s; will ' 'replicate out and remove.' % (object_file, name, bpart)) except (Exception, Timeout) as e: if 'no such table' in str(e): self.logger.error(_('Quarantining DB %s'), object_file) quarantine_db(broker.db_file, broker.db_type) else: self.logger.exception(_('ERROR reading db %s'), object_file) self.stats['failure'] += 1 self.logger.increment('failures') return # The db is considered deleted if the delete_timestamp value is greater # than the put_timestamp, and there are no objects. delete_timestamp = 0 try: delete_timestamp = float(info['delete_timestamp']) except ValueError: pass put_timestamp = 0 try: put_timestamp = float(info['put_timestamp']) except ValueError: pass if delete_timestamp < (now - self.reclaim_age) and \ delete_timestamp > put_timestamp and \ info['count'] in (None, '', 0, '0'): if self.report_up_to_date(full_info): self.delete_db(object_file) self.logger.timing_since('timing', start_time) return responses = [] nodes = self.ring.get_part_nodes(int(partition)) if shouldbehere: shouldbehere = bool([n for n in nodes if n['id'] == node_id]) # See Footnote [1] for an explanation of the repl_nodes assignment. i = 0 while i < len(nodes) and nodes[i]['id'] != node_id: i += 1 repl_nodes = nodes[i + 1:] + nodes[:i] more_nodes = self.ring.get_more_nodes(int(partition)) for node in repl_nodes: success = False try: success = self._repl_to_node(node, broker, partition, info) except DriveNotMounted: repl_nodes.append(more_nodes.next()) self.logger.error(_('ERROR Remote drive not mounted %s'), node) except (Exception, Timeout): self.logger.exception(_('ERROR syncing %(file)s with node' ' %(node)s'), {'file': object_file, 'node': node}) self.stats['success' if success else 'failure'] += 1 self.logger.increment('successes' if success else 'failures') responses.append(success) if not shouldbehere and all(responses): # If the db shouldn't be on this node and has been successfully # synced to all of its peers, it can be removed. self.delete_db(object_file) self.logger.timing_since('timing', start_time) def delete_db(self, object_file): hash_dir = os.path.dirname(object_file) suf_dir = os.path.dirname(hash_dir) with lock_parent_directory(object_file): shutil.rmtree(hash_dir, True) try: os.rmdir(suf_dir) except OSError as err: if err.errno not in (errno.ENOENT, errno.ENOTEMPTY): self.logger.exception( _('ERROR while trying to clean up %s') % suf_dir) self.stats['remove'] += 1 device_name = self.extract_device(object_file) self.logger.increment('removes.' + device_name) def extract_device(self, object_file): """ Extract the device name from an object path. Returns "UNKNOWN" if the path could not be extracted successfully for some reason. :param object_file: the path to a database file. """ match = self.extract_device_re.match(object_file) if match: return match.groups()[0] return "UNKNOWN" def report_up_to_date(self, full_info): return True def run_once(self, *args, **kwargs): """Run a replication pass once.""" self._zero_stats() dirs = [] ips = whataremyips() if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return for node in self.ring.devs: if (node and node['replication_ip'] in ips and node['replication_port'] == self.port): if self.mount_check and not ismount( os.path.join(self.root, node['device'])): self.logger.warn( _('Skipping %(device)s as it is not mounted') % node) continue unlink_older_than( os.path.join(self.root, node['device'], 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): dirs.append((datadir, node['id'])) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in roundrobin_datadirs(dirs): self.cpool.spawn_n( self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) self._report_stats() def run_forever(self, *args, **kwargs): """ Replicate dbs under the given root in an infinite loop. """ sleep(random.random() * self.interval) while True: begin = time.time() try: self.run_once() except (Exception, Timeout): self.logger.exception(_('ERROR trying to replicate')) elapsed = time.time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) class ReplicatorRpc(object): """Handle Replication RPC calls. TODO(redbo): document please :)""" def __init__(self, root, datadir, broker_class, mount_check=True, logger=None): self.root = root self.datadir = datadir self.broker_class = broker_class self.mount_check = mount_check self.logger = logger or get_logger({}, log_route='replicator-rpc') def dispatch(self, replicate_args, args): if not hasattr(args, 'pop'): return HTTPBadRequest(body='Invalid object type') op = args.pop(0) drive, partition, hsh = replicate_args if self.mount_check and not ismount(os.path.join(self.root, drive)): return Response(status='507 %s is not mounted' % drive) db_file = os.path.join(self.root, drive, storage_directory(self.datadir, partition, hsh), hsh + '.db') if op == 'rsync_then_merge': return self.rsync_then_merge(drive, db_file, args) if op == 'complete_rsync': return self.complete_rsync(drive, db_file, args) else: # someone might be about to rsync a db to us, # make sure there's a tmp dir to receive it. mkdirs(os.path.join(self.root, drive, 'tmp')) if not os.path.exists(db_file): return HTTPNotFound() return getattr(self, op)(self.broker_class(db_file), args) def sync(self, broker, args): (remote_sync, hash_, id_, created_at, put_timestamp, delete_timestamp, metadata) = args timemark = time.time() try: info = broker.get_replication_info() except (Exception, Timeout) as e: if 'no such table' in str(e): self.logger.error(_("Quarantining DB %s"), broker) quarantine_db(broker.db_file, broker.db_type) return HTTPNotFound() raise timespan = time.time() - timemark if timespan > DEBUG_TIMINGS_THRESHOLD: self.logger.debug(_('replicator-rpc-sync time for info: %.02fs') % timespan) if metadata: timemark = time.time() broker.update_metadata(simplejson.loads(metadata)) timespan = time.time() - timemark if timespan > DEBUG_TIMINGS_THRESHOLD: self.logger.debug(_('replicator-rpc-sync time for ' 'update_metadata: %.02fs') % timespan) if info['put_timestamp'] != put_timestamp or \ info['created_at'] != created_at or \ info['delete_timestamp'] != delete_timestamp: timemark = time.time() broker.merge_timestamps( created_at, put_timestamp, delete_timestamp) timespan = time.time() - timemark if timespan > DEBUG_TIMINGS_THRESHOLD: self.logger.debug(_('replicator-rpc-sync time for ' 'merge_timestamps: %.02fs') % timespan) timemark = time.time() info['point'] = broker.get_sync(id_) timespan = time.time() - timemark if timespan > DEBUG_TIMINGS_THRESHOLD: self.logger.debug(_('replicator-rpc-sync time for get_sync: ' '%.02fs') % timespan) if hash_ == info['hash'] and info['point'] < remote_sync: timemark = time.time() broker.merge_syncs([{'remote_id': id_, 'sync_point': remote_sync}]) info['point'] = remote_sync timespan = time.time() - timemark if timespan > DEBUG_TIMINGS_THRESHOLD: self.logger.debug(_('replicator-rpc-sync time for ' 'merge_syncs: %.02fs') % timespan) return Response(simplejson.dumps(info)) def merge_syncs(self, broker, args): broker.merge_syncs(args[0]) return HTTPAccepted() def merge_items(self, broker, args): broker.merge_items(args[0], args[1]) return HTTPAccepted() def complete_rsync(self, drive, db_file, args): old_filename = os.path.join(self.root, drive, 'tmp', args[0]) if os.path.exists(db_file): return HTTPNotFound() if not os.path.exists(old_filename): return HTTPNotFound() broker = self.broker_class(old_filename) broker.newid(args[0]) renamer(old_filename, db_file) return HTTPNoContent() def rsync_then_merge(self, drive, db_file, args): old_filename = os.path.join(self.root, drive, 'tmp', args[0]) if not os.path.exists(db_file) or not os.path.exists(old_filename): return HTTPNotFound() new_broker = self.broker_class(old_filename) existing_broker = self.broker_class(db_file) point = -1 objects = existing_broker.get_items_since(point, 1000) while len(objects): new_broker.merge_items(objects) point = objects[-1]['ROWID'] objects = existing_broker.get_items_since(point, 1000) sleep() new_broker.newid(args[0]) renamer(old_filename, db_file) return HTTPNoContent() # Footnote [1]: # This orders the nodes so that, given nodes a b c, a will contact b then c, # b will contact c then a, and c will contact a then b -- in other words, each # node will always contact the next node in the list first. # This helps in the case where databases are all way out of sync, so each # node is likely to be sending to a different node than it's receiving from, # rather than two nodes talking to each other, starving out the third. # If the third didn't even have a copy and the first two nodes were way out # of sync, such starvation would mean the third node wouldn't get any copy # until the first two nodes finally got in sync, which could take a while. # This new ordering ensures such starvation doesn't occur, making the data # more durable. swift-1.13.1/swift/container/0000775000175400017540000000000012323703665017214 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/container/replicator.py0000664000175400017540000000214512323703611021723 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.container.backend import ContainerBroker, DATADIR from swift.common import db_replicator class ContainerReplicator(db_replicator.Replicator): server_type = 'container' brokerclass = ContainerBroker datadir = DATADIR default_port = 6001 def report_up_to_date(self, full_info): for key in ('put_timestamp', 'delete_timestamp', 'object_count', 'bytes_used'): if full_info['reported_' + key] != full_info[key]: return False return True swift-1.13.1/swift/container/updater.py0000664000175400017540000003063612323703611021231 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import signal import sys import time from swift import gettext_ as _ from random import random, shuffle from tempfile import mkstemp from eventlet import spawn, patcher, Timeout import swift.common.db from swift.container.backend import ContainerBroker, DATADIR from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, config_true_value, ismount, \ dump_recon_cache, quorum_size from swift.common.daemon import Daemon from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR class ContainerUpdater(Daemon): """Update container information in account listings.""" def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='container-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 300)) self.account_ring = None self.concurrency = int(conf.get('concurrency', 4)) self.slowdown = float(conf.get('slowdown', 0.01)) self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.no_changes = 0 self.successes = 0 self.failures = 0 self.account_suppressions = {} self.account_suppression_time = \ float(conf.get('account_suppression_time', 60)) self.new_account_suppressions = None swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "container.recon") self.user_agent = 'container-updater %s' % os.getpid() def get_account_ring(self): """Get the account ring. Load it if it hasn't been yet.""" if not self.account_ring: self.account_ring = Ring(self.swift_dir, ring_name='account') return self.account_ring def get_paths(self): """ Get paths to all of the partitions on each drive to be processed. :returns: a list of paths """ paths = [] for device in os.listdir(self.devices): dev_path = os.path.join(self.devices, device) if self.mount_check and not ismount(dev_path): self.logger.warn(_('%s is not mounted'), device) continue con_path = os.path.join(dev_path, DATADIR) if not os.path.exists(con_path): continue for partition in os.listdir(con_path): paths.append(os.path.join(con_path, partition)) shuffle(paths) return paths def _load_suppressions(self, filename): try: with open(filename, 'r') as tmpfile: for line in tmpfile: account, until = line.split() until = float(until) self.account_suppressions[account] = until except Exception: self.logger.exception( _('ERROR with loading suppressions from %s: ') % filename) finally: os.unlink(filename) def run_forever(self, *args, **kwargs): """ Run the updator continuously. """ time.sleep(random() * self.interval) while True: self.logger.info(_('Begin container update sweep')) begin = time.time() now = time.time() expired_suppressions = \ [a for a, u in self.account_suppressions.iteritems() if u < now] for account in expired_suppressions: del self.account_suppressions[account] pid2filename = {} # read from account ring to ensure it's fresh self.get_account_ring().get_nodes('') for path in self.get_paths(): while len(pid2filename) >= self.concurrency: pid = os.wait()[0] try: self._load_suppressions(pid2filename[pid]) finally: del pid2filename[pid] fd, tmpfilename = mkstemp() os.close(fd) pid = os.fork() if pid: pid2filename[pid] = tmpfilename else: signal.signal(signal.SIGTERM, signal.SIG_DFL) patcher.monkey_patch(all=False, socket=True) self.no_changes = 0 self.successes = 0 self.failures = 0 self.new_account_suppressions = open(tmpfilename, 'w') forkbegin = time.time() self.container_sweep(path) elapsed = time.time() - forkbegin self.logger.debug( _('Container update sweep of %(path)s completed: ' '%(elapsed).02fs, %(success)s successes, %(fail)s ' 'failures, %(no_change)s with no changes'), {'path': path, 'elapsed': elapsed, 'success': self.successes, 'fail': self.failures, 'no_change': self.no_changes}) sys.exit() while pid2filename: pid = os.wait()[0] try: self._load_suppressions(pid2filename[pid]) finally: del pid2filename[pid] elapsed = time.time() - begin self.logger.info(_('Container update sweep completed: %.02fs'), elapsed) dump_recon_cache({'container_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Run the updater once. """ patcher.monkey_patch(all=False, socket=True) self.logger.info(_('Begin container update single threaded sweep')) begin = time.time() self.no_changes = 0 self.successes = 0 self.failures = 0 for path in self.get_paths(): self.container_sweep(path) elapsed = time.time() - begin self.logger.info(_( 'Container update single threaded sweep completed: ' '%(elapsed).02fs, %(success)s successes, %(fail)s failures, ' '%(no_change)s with no changes'), {'elapsed': elapsed, 'success': self.successes, 'fail': self.failures, 'no_change': self.no_changes}) dump_recon_cache({'container_updater_sweep': elapsed}, self.rcache, self.logger) def container_sweep(self, path): """ Walk the path looking for container DBs and process them. :param path: path to walk """ for root, dirs, files in os.walk(path): for file in files: if file.endswith('.db'): self.process_container(os.path.join(root, file)) time.sleep(self.slowdown) def process_container(self, dbfile): """ Process a container, and update the information in the account. :param dbfile: container DB to process """ start_time = time.time() broker = ContainerBroker(dbfile, logger=self.logger) info = broker.get_info() # Don't send updates if the container was auto-created since it # definitely doesn't have up to date statistics. if float(info['put_timestamp']) <= 0: return if self.account_suppressions.get(info['account'], 0) > time.time(): return if info['put_timestamp'] > info['reported_put_timestamp'] or \ info['delete_timestamp'] > info['reported_delete_timestamp'] \ or info['object_count'] != info['reported_object_count'] or \ info['bytes_used'] != info['reported_bytes_used']: container = '/%s/%s' % (info['account'], info['container']) part, nodes = self.get_account_ring().get_nodes(info['account']) events = [spawn(self.container_report, node, part, container, info['put_timestamp'], info['delete_timestamp'], info['object_count'], info['bytes_used']) for node in nodes] successes = 0 for event in events: if is_success(event.wait()): successes += 1 if successes >= quorum_size(len(events)): self.logger.increment('successes') self.successes += 1 self.logger.debug( _('Update report sent for %(container)s %(dbfile)s'), {'container': container, 'dbfile': dbfile}) broker.reported(info['put_timestamp'], info['delete_timestamp'], info['object_count'], info['bytes_used']) else: self.logger.increment('failures') self.failures += 1 self.logger.debug( _('Update report failed for %(container)s %(dbfile)s'), {'container': container, 'dbfile': dbfile}) self.account_suppressions[info['account']] = until = \ time.time() + self.account_suppression_time if self.new_account_suppressions: print >>self.new_account_suppressions, \ info['account'], until # Only track timing data for attempted updates: self.logger.timing_since('timing', start_time) else: self.logger.increment('no_changes') self.no_changes += 1 def container_report(self, node, part, container, put_timestamp, delete_timestamp, count, bytes): """ Report container info to an account server. :param node: node dictionary from the account ring :param part: partition the account is on :param container: container name :param put_timestamp: put timestamp :param delete_timestamp: delete timestamp :param count: object count in the container :param bytes: bytes used in the container """ with ConnectionTimeout(self.conn_timeout): try: headers = { 'X-Put-Timestamp': put_timestamp, 'X-Delete-Timestamp': delete_timestamp, 'X-Object-Count': count, 'X-Bytes-Used': bytes, 'X-Account-Override-Deleted': 'yes', 'user-agent': self.user_agent} conn = http_connect( node['ip'], node['port'], node['device'], part, 'PUT', container, headers=headers) except (Exception, Timeout): self.logger.exception(_( 'ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later): '), node) return HTTP_INTERNAL_SERVER_ERROR with Timeout(self.node_timeout): try: resp = conn.getresponse() resp.read() return resp.status except (Exception, Timeout): if self.logger.getEffectiveLevel() <= logging.DEBUG: self.logger.exception( _('Exception with %(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR finally: conn.close() swift-1.13.1/swift/container/server.py0000664000175400017540000006003412323703614021071 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time import traceback from datetime import datetime from swift import gettext_ as _ from xml.etree.cElementTree import Element, SubElement, tostring from eventlet import Timeout import swift.common.db from swift.container.backend import ContainerBroker, DATADIR from swift.common.db import DatabaseAlreadyExists from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.request_helpers import get_param, get_listing_content_type, \ split_and_validate_path, is_sys_or_user_meta from swift.common.utils import get_logger, hash_path, public, \ normalize_timestamp, storage_directory, validate_sync_to, \ config_true_value, json, timing_stats, replication, \ override_bytes_from_content_type from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.db_replicator import ReplicatorRpc from swift.common.http import HTTP_NOT_FOUND, is_success from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ HTTPCreated, HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \ HTTPInsufficientStorage, HTTPException, HeaderKeyDict class ContainerController(object): """WSGI Controller for the container server.""" # Ensure these are all lowercase save_headers = ['x-container-read', 'x-container-write', 'x-container-sync-key', 'x-container-sync-to'] def __init__(self, conf, logger=None): self.logger = logger or get_logger(conf, log_route='container-server') self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.root = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join( conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] self.replicator_rpc = ReplicatorRpc( self.root, DATADIR, ContainerBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' if config_true_value(conf.get('allow_versions', 'f')): self.save_headers.append('x-versions-location') swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) def _get_container_broker(self, drive, part, account, container, **kwargs): """ Get a DB broker for the container. :param drive: drive that holds the container :param part: partition the container is in :param account: account name :param container: container name :returns: ContainerBroker object """ hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') kwargs.setdefault('account', account) kwargs.setdefault('container', container) kwargs.setdefault('logger', self.logger) return ContainerBroker(db_path, **kwargs) def account_update(self, req, account, container, broker): """ Update the account server(s) with latest container info. :param req: swob.Request object :param account: account name :param container: container name :param broker: container DB broker object :returns: if all the account requests return a 404 error code, HTTPNotFound response object, if the account cannot be updated due to a malformed header, an HTTPBadRequest response object, otherwise None. """ account_hosts = [h.strip() for h in req.headers.get('X-Account-Host', '').split(',')] account_devices = [d.strip() for d in req.headers.get('X-Account-Device', '').split(',')] account_partition = req.headers.get('X-Account-Partition', '') if len(account_hosts) != len(account_devices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error(_('ERROR Account update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (req.headers.get('X-Account-Host', ''), req.headers.get('X-Account-Device', ''))) return HTTPBadRequest(req=req) if account_partition: updates = zip(account_hosts, account_devices) else: updates = [] account_404s = 0 for account_host, account_device in updates: account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = HeaderKeyDict({ 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], 'x-trans-id': req.headers.get('x-trans-id', '-'), 'user-agent': 'container-server %s' % os.getpid(), 'referer': req.as_referer()}) if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' try: with ConnectionTimeout(self.conn_timeout): conn = http_connect( account_ip, account_port, account_device, account_partition, 'PUT', new_path, account_headers) with Timeout(self.node_timeout): account_response = conn.getresponse() account_response.read() if account_response.status == HTTP_NOT_FOUND: account_404s += 1 elif not is_success(account_response.status): self.logger.error(_( 'ERROR Account update failed ' 'with %(ip)s:%(port)s/%(device)s (will retry ' 'later): Response %(status)s %(reason)s'), {'ip': account_ip, 'port': account_port, 'device': account_device, 'status': account_response.status, 'reason': account_response.reason}) except (Exception, Timeout): self.logger.exception(_( 'ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later)'), {'ip': account_ip, 'port': account_port, 'device': account_device}) if updates and account_404s == len(updates): return HTTPNotFound(req=req) else: return None @public @timing_stats() def DELETE(self, req): """Handle HTTP DELETE request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if account.startswith(self.auto_create_account_prefix) and obj and \ not os.path.exists(broker.db_file): try: broker.initialize(normalize_timestamp( req.headers.get('x-timestamp') or time.time())) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() if obj: # delete object broker.delete_object(obj, req.headers.get('x-timestamp')) return HTTPNoContent(request=req) else: # delete container if not broker.empty(): return HTTPConflict(request=req) existed = float(broker.get_info()['put_timestamp']) and \ not broker.is_deleted() broker.delete_db(req.headers['X-Timestamp']) if not broker.is_deleted(): return HTTPConflict(request=req) resp = self.account_update(req, account, container, broker) if resp: return resp if existed: return HTTPNoContent(request=req) return HTTPNotFound() def _update_or_create(self, req, broker, timestamp): if not os.path.exists(broker.db_file): try: broker.initialize(timestamp) except DatabaseAlreadyExists: pass else: return True # created created = broker.is_deleted() broker.update_put_timestamp(timestamp) if broker.is_deleted(): raise HTTPConflict(request=req) return created @public @timing_stats() def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) timestamp = normalize_timestamp(req.headers['x-timestamp']) broker = self._get_container_broker(drive, part, account, container) if obj: # put container object if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp) except DatabaseAlreadyExists: pass if not os.path.exists(broker.db_file): return HTTPNotFound() broker.put_object(obj, timestamp, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag']) return HTTPCreated(request=req) else: # put container created = self._update_or_create(req, broker, timestamp) metadata = {} metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata) resp = self.account_update(req, account, container, broker) if resp: return resp if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req) @public @timing_stats(sample_rate=0.1) def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } headers.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type return HTTPNoContent(request=req, headers=headers, charset='utf-8') def update_data_record(self, record): """ Perform any mutations to container listing records that are common to all serialization formats, and returns it as a dict. Converts created time to iso timestamp. Replaces size with 'swift_bytes' content type parameter. :params record: object entry record :returns: modified record """ (name, created, size, content_type, etag) = record if content_type is None: return {'subdir': name} response = {'bytes': size, 'hash': etag, 'name': name, 'content_type': content_type} last_modified = datetime.utcfromtimestamp(float(created)).isoformat() # python isoformat() doesn't include msecs when zero if len(last_modified) < len("1970-01-01T00:00:00.000000"): last_modified += ".000000" response['last_modified'] = last_modified override_bytes_from_content_type(response, logger=self.logger) return response @public @timing_stats() def GET(self, req): """Handle HTTP GET request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed( request=req, body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key)): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps([self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in ["name", "hash", "bytes", "content_type", "last_modified"]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "", '', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret @public @replication @timing_stats(sample_rate=0.01) def REPLICATE(self, req): """ Handle HTTP REPLICATE request (json-encoded RPC calls for replication.) """ post_args = split_and_validate_path(req, 3) drive, partition, hash = post_args if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) try: args = json.load(req.environ['wsgi.input']) except ValueError as err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) ret.request = req return ret @public @timing_stats() def POST(self, req): """Handle HTTP POST request.""" drive, part, account, container = split_and_validate_path(req, 4) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing or bad timestamp', request=req, content_type='text/plain') if 'x-container-sync-to' in req.headers: err, sync_to, realm, realm_key = validate_sync_to( req.headers['x-container-sync-to'], self.allowed_sync_hosts, self.realms_conf) if err: return HTTPBadRequest(err) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if broker.is_deleted(): return HTTPNotFound(request=req) timestamp = normalize_timestamp(req.headers['x-timestamp']) metadata = {} metadata.update( (key, (value, timestamp)) for key, value in req.headers.iteritems() if key.lower() in self.save_headers or is_sys_or_user_meta('container', key)) if metadata: if 'X-Container-Sync-To' in metadata: if 'X-Container-Sync-To' not in broker.metadata or \ metadata['X-Container-Sync-To'][0] != \ broker.metadata['X-Container-Sync-To'][0]: broker.set_x_container_sync_points(-1, -1) broker.update_metadata(metadata) return HTTPNoContent(request=req) def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) if self.log_requests: log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response) def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI container server apps""" conf = global_conf.copy() conf.update(local_conf) return ContainerController(conf) swift-1.13.1/swift/container/auditor.py0000664000175400017540000001225012323703611021224 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time from swift import gettext_ as _ from random import random from eventlet import Timeout import swift.common.db from swift.container.backend import ContainerBroker, DATADIR from swift.common.utils import get_logger, audit_location_generator, \ config_true_value, dump_recon_cache, ratelimit_sleep from swift.common.daemon import Daemon class ContainerAuditor(Daemon): """Audit containers.""" def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='container-auditor') self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.interval = int(conf.get('interval', 1800)) self.container_passes = 0 self.container_failures = 0 self.containers_running_time = 0 self.max_containers_per_second = \ float(conf.get('containers_per_second', 200)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "container.recon") def _one_audit_pass(self, reported): all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.container_audit(path) if time.time() - reported >= 3600: # once an hour self.logger.info( _('Since %(time)s: Container audits: %(pass)s passed ' 'audit, %(fail)s failed audit'), {'time': time.ctime(reported), 'pass': self.container_passes, 'fail': self.container_failures}) dump_recon_cache( {'container_audits_since': reported, 'container_audits_passed': self.container_passes, 'container_audits_failed': self.container_failures}, self.rcache, self.logger) reported = time.time() self.container_passes = 0 self.container_failures = 0 self.containers_running_time = ratelimit_sleep( self.containers_running_time, self.max_containers_per_second) return reported def run_forever(self, *args, **kwargs): """Run the container audit until stopped.""" reported = time.time() time.sleep(random() * self.interval) while True: self.logger.info(_('Begin container audit pass.')) begin = time.time() try: reported = self._one_audit_pass(reported) except (Exception, Timeout): self.logger.increment('errors') self.logger.exception(_('ERROR auditing')) elapsed = time.time() - begin if elapsed < self.interval: time.sleep(self.interval - elapsed) self.logger.info( _('Container audit pass completed: %.02fs'), elapsed) dump_recon_cache({'container_auditor_pass_completed': elapsed}, self.rcache, self.logger) def run_once(self, *args, **kwargs): """Run the container audit once.""" self.logger.info(_('Begin container audit "once" mode')) begin = reported = time.time() self._one_audit_pass(reported) elapsed = time.time() - begin self.logger.info( _('Container audit "once" mode completed: %.02fs'), elapsed) dump_recon_cache({'container_auditor_pass_completed': elapsed}, self.rcache, self.logger) def container_audit(self, path): """ Audits the given container path :param path: the path to a container db """ start_time = time.time() try: broker = ContainerBroker(path) if not broker.is_deleted(): broker.get_info() self.logger.increment('passes') self.container_passes += 1 self.logger.debug(_('Audit passed for %s'), broker) except (Exception, Timeout): self.logger.increment('failures') self.container_failures += 1 self.logger.exception(_('ERROR Could not get container info %s'), path) self.logger.timing_since('timing', start_time) swift-1.13.1/swift/container/sync.py0000664000175400017540000005174412323703614020547 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import uuid from swift import gettext_ as _ from time import ctime, time from random import choice, random, shuffle from struct import unpack_from from eventlet import sleep, Timeout import swift.common.db from swift.container.backend import ContainerBroker, DATADIR from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.direct_client import direct_get_object from swift.common.internal_client import delete_object, put_object from swift.common.exceptions import ClientException from swift.common.ring import Ring from swift.common.utils import audit_location_generator, get_logger, \ hash_path, config_true_value, validate_sync_to, whataremyips, \ FileLikeIter, urlparse, quote from swift.common.daemon import Daemon from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND class ContainerSync(Daemon): """ Daemon to sync syncable containers. This is done by scanning the local devices for container databases and checking for x-container-sync-to and x-container-sync-key metadata values. If they exist, newer rows since the last sync will trigger PUTs or DELETEs to the other container. .. note:: Container sync will sync object POSTs only if the proxy server is set to use "object_post_as_copy = true" which is the default. So-called fast object posts, "object_post_as_copy = false" do not update the container listings and therefore can't be detected for synchronization. The actual syncing is slightly more complicated to make use of the three (or number-of-replicas) main nodes for a container without each trying to do the exact same work but also without missing work if one node happens to be down. Two sync points are kept per container database. All rows between the two sync points trigger updates. Any rows newer than both sync points cause updates depending on the node's position for the container (primary nodes do one third, etc. depending on the replica count of course). After a sync run, the first sync point is set to the newest ROWID known and the second sync point is set to newest ROWID for which all updates have been sent. An example may help. Assume replica count is 3 and perfectly matching ROWIDs starting at 1. First sync run, database has 6 rows: * SyncPoint1 starts as -1. * SyncPoint2 starts as -1. * No rows between points, so no "all updates" rows. * Six rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 6 (the newest ROWID known). * SyncPoint2 is left as -1 since no "all updates" rows were synced. Next sync run, database has 12 rows: * SyncPoint1 starts as 6. * SyncPoint2 starts as -1. * The rows between -1 and 6 all trigger updates (most of which should short-circuit on the remote end as having already been done). * Six more rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 12 (the newest ROWID known). * SyncPoint2 is set as 6 (the newest "all updates" ROWID). In this way, under normal circumstances each node sends its share of updates each run and just sends a batch of older updates to ensure nothing was missed. :param conf: The dict of configuration values from the [container-sync] section of the container-server.conf :param container_ring: If None, the /container.ring.gz will be loaded. This is overridden by unit tests. :param object_ring: If None, the /object.ring.gz will be loaded. This is overridden by unit tests. """ def __init__(self, conf, container_ring=None, object_ring=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join( conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip()] #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers that didn't have sync turned on. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Time of last stats report. self.reported = time() swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(swift_dir, ring_name='container') #: swift.common.ring.Ring for locating objects. self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) def run_forever(self, *args, **kwargs): """ Runs container sync scans until stopped. """ sleep(random() * self.interval) while True: begin = time() all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() elapsed = time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Runs a single container sync scan. """ self.logger.info(_('Begin container sync "once" mode')) begin = time() all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() self.report() elapsed = time() - begin self.logger.info( _('Container sync "once" mode completed: %.02fs'), elapsed) def report(self): """ Writes a report of the stats to the logger and resets the stats for the next report. """ self.logger.info( _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s ' 'puts], %(skip)s skipped, %(fail)s failed'), {'time': ctime(self.reported), 'sync': self.container_syncs, 'delete': self.container_deletes, 'put': self.container_puts, 'skip': self.container_skips, 'fail': self.container_failures}) self.reported = time() self.container_syncs = 0 self.container_deletes = 0 self.container_puts = 0 self.container_skips = 0 self.container_failures = 0 def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path) info = broker.get_info() x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if node['ip'] in self._myips and node['port'] == self._myport: break else: return if not broker.is_deleted(): sync_to = None user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': user_key = value if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return err, sync_to, realm, realm_key = validate_sync_to( sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), {'db_file': str(broker), 'validate_sync_to_err': err}) self.container_failures += 1 self.logger.increment('failures') return stop_at = time() + self.container_time next_sync_point = None while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.) and will skip # problematic rows as needed in case of faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the nodes # didn't succeed. if not self.container_sync_row( row, sync_to, user_key, broker, info, realm, realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points(None, next_sync_point) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed or in case it failed to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row( row, sync_to, user_key, broker, info, realm, realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 self.logger.increment('syncs') except (Exception, Timeout) as err: self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker if broker else path) def container_sync_row(self, row, sync_to, user_key, broker, info, realm, realm_key): """ Sends the update the row indicates to the sync_to container. :param row: The updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param broker: The local container database broker. :param info: The get_info result from the local container database broker. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :returns: True on success """ try: start_time = time() if row['deleted']: try: headers = {'x-timestamp': row['created_at']} if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote( row['name']) sig = self.realms_conf.get_sig( 'DELETE', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key delete_object(sync_to, name=row['name'], headers=headers, proxy=self.select_http_proxy()) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise self.container_deletes += 1 self.logger.increment('deletes') self.logger.timing_since('deletes.timing', start_time) else: part, nodes = self.object_ring.get_nodes( info['account'], info['container'], row['name']) shuffle(nodes) exc = None looking_for_timestamp = float(row['created_at']) timestamp = -1 headers = body = None for node in nodes: try: these_headers, this_body = direct_get_object( node, part, info['account'], info['container'], row['name'], resp_chunk_size=65536) this_timestamp = float(these_headers['x-timestamp']) if this_timestamp > timestamp: timestamp = this_timestamp headers = these_headers body = this_body except ClientException as err: # If any errors are not 404, make sure we report the # non-404 one. We don't want to mistakenly assume the # object no longer exists just because one says so and # the others errored for some other reason. if not exc or exc.http_status == HTTP_NOT_FOUND: exc = err except (Exception, Timeout) as err: exc = err if timestamp < looking_for_timestamp: if exc: raise exc raise Exception( _('Unknown exception trying to GET: %(node)r ' '%(account)r %(container)r %(object)r'), {'node': node, 'part': part, 'account': info['account'], 'container': info['container'], 'object': row['name']}) for key in ('date', 'last-modified'): if key in headers: del headers[key] if 'etag' in headers: headers['etag'] = headers['etag'].strip('"') headers['x-timestamp'] = row['created_at'] if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote(row['name']) sig = self.realms_conf.get_sig( 'PUT', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), proxy=self.select_http_proxy()) self.container_puts += 1 self.logger.increment('puts') self.logger.timing_since('puts.timing', start_time) except ClientException as err: if err.http_status == HTTP_UNAUTHORIZED: self.logger.info( _('Unauth %(sync_from)r => %(sync_to)r'), {'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to}) elif err.http_status == HTTP_NOT_FOUND: self.logger.info( _('Not found %(sync_from)r => %(sync_to)r \ - object %(obj_name)r'), {'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to, 'obj_name': row['name']}) else: self.logger.exception( _('ERROR Syncing %(db_file)s %(row)s'), {'db_file': str(broker), 'row': row}) self.container_failures += 1 self.logger.increment('failures') return False except (Exception, Timeout) as err: self.logger.exception( _('ERROR Syncing %(db_file)s %(row)s'), {'db_file': str(broker), 'row': row}) self.container_failures += 1 self.logger.increment('failures') return False return True def select_http_proxy(self): return choice(self.http_proxies) if self.http_proxies else None swift-1.13.1/swift/container/backend.py0000664000175400017540000005036612323703611021156 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Pluggable Back-ends for Container Server """ import os from uuid import uuid4 import time import cPickle as pickle import errno import sqlite3 from swift.common.utils import normalize_timestamp, lock_parent_directory from swift.common.db import DatabaseBroker, DatabaseConnectionError, \ PENDING_CAP, PICKLE_PROTOCOL, utf8encode DATADIR = 'containers' class ContainerBroker(DatabaseBroker): """Encapsulates working with a container database.""" db_type = 'container' db_contains_type = 'object' db_reclaim_timestamp = 'created_at' def _initialize(self, conn, put_timestamp): """ Create a brand new container database (tables, indices, triggers, etc.) """ if not self.account: raise ValueError( 'Attempting to create a new database with no account set') if not self.container: raise ValueError( 'Attempting to create a new database with no container set') self.create_object_table(conn) self.create_container_stat_table(conn, put_timestamp) def create_object_table(self, conn): """ Create the object table which is specifc to the container DB. Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object """ conn.executescript(""" CREATE TABLE object ( ROWID INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, created_at TEXT, size INTEGER, content_type TEXT, etag TEXT, deleted INTEGER DEFAULT 0 ); CREATE INDEX ix_object_deleted_name ON object (deleted, name); CREATE TRIGGER object_insert AFTER INSERT ON object BEGIN UPDATE container_stat SET object_count = object_count + (1 - new.deleted), bytes_used = bytes_used + new.size, hash = chexor(hash, new.name, new.created_at); END; CREATE TRIGGER object_update BEFORE UPDATE ON object BEGIN SELECT RAISE(FAIL, 'UPDATE not allowed; DELETE and INSERT'); END; CREATE TRIGGER object_delete AFTER DELETE ON object BEGIN UPDATE container_stat SET object_count = object_count - (1 - old.deleted), bytes_used = bytes_used - old.size, hash = chexor(hash, old.name, old.created_at); END; """) def create_container_stat_table(self, conn, put_timestamp=None): """ Create the container_stat table which is specific to the container DB. Not a part of Pluggable Back-ends, internal to the baseline code. :param conn: DB connection object :param put_timestamp: put timestamp """ if put_timestamp is None: put_timestamp = normalize_timestamp(0) conn.executescript(""" CREATE TABLE container_stat ( account TEXT, container TEXT, created_at TEXT, put_timestamp TEXT DEFAULT '0', delete_timestamp TEXT DEFAULT '0', object_count INTEGER, bytes_used INTEGER, reported_put_timestamp TEXT DEFAULT '0', reported_delete_timestamp TEXT DEFAULT '0', reported_object_count INTEGER DEFAULT 0, reported_bytes_used INTEGER DEFAULT 0, hash TEXT default '00000000000000000000000000000000', id TEXT, status TEXT DEFAULT '', status_changed_at TEXT DEFAULT '0', metadata TEXT DEFAULT '', x_container_sync_point1 INTEGER DEFAULT -1, x_container_sync_point2 INTEGER DEFAULT -1 ); INSERT INTO container_stat (object_count, bytes_used) VALUES (0, 0); """) conn.execute(''' UPDATE container_stat SET account = ?, container = ?, created_at = ?, id = ?, put_timestamp = ? ''', (self.account, self.container, normalize_timestamp(time.time()), str(uuid4()), put_timestamp)) def get_db_version(self, conn): if self._db_version == -1: self._db_version = 0 for row in conn.execute(''' SELECT name FROM sqlite_master WHERE name = 'ix_object_deleted_name' '''): self._db_version = 1 return self._db_version def _newid(self, conn): conn.execute(''' UPDATE container_stat SET reported_put_timestamp = 0, reported_delete_timestamp = 0, reported_object_count = 0, reported_bytes_used = 0''') def _delete_db(self, conn, timestamp): """ Mark the DB as deleted :param conn: DB connection object :param timestamp: timestamp to mark as deleted """ conn.execute(""" UPDATE container_stat SET delete_timestamp = ?, status = 'DELETED', status_changed_at = ? WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp)) def _commit_puts_load(self, item_list, entry): """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`""" (name, timestamp, size, content_type, etag, deleted) = \ pickle.loads(entry.decode('base64')) item_list.append({'name': name, 'created_at': timestamp, 'size': size, 'content_type': content_type, 'etag': etag, 'deleted': deleted}) def empty(self): """ Check if container DB is empty. :returns: True if the database has no active objects, False otherwise """ self._commit_puts_stale_ok() with self.get() as conn: row = conn.execute( 'SELECT object_count from container_stat').fetchone() return (row[0] == 0) def delete_object(self, name, timestamp): """ Mark an object deleted. :param name: object name to be deleted :param timestamp: timestamp when the object was marked as deleted """ self.put_object(name, timestamp, 0, 'application/deleted', 'noetag', 1) def put_object(self, name, timestamp, size, content_type, etag, deleted=0): """ Creates an object in the DB with its metadata. :param name: object name to be created :param timestamp: timestamp of when the object was created :param size: object size :param content_type: object content-type :param etag: object etag :param deleted: if True, marks the object as deleted and sets the deteleted_at timestamp to timestamp """ record = {'name': name, 'created_at': timestamp, 'size': size, 'content_type': content_type, 'etag': etag, 'deleted': deleted} if self.db_file == ':memory:': self.merge_items([record]) return if not os.path.exists(self.db_file): raise DatabaseConnectionError(self.db_file, "DB doesn't exist") pending_size = 0 try: pending_size = os.path.getsize(self.pending_file) except OSError as err: if err.errno != errno.ENOENT: raise if pending_size > PENDING_CAP: self._commit_puts([record]) else: with lock_parent_directory(self.pending_file, self.pending_timeout): with open(self.pending_file, 'a+b') as fp: # Colons aren't used in base64 encoding; so they are our # delimiter fp.write(':') fp.write(pickle.dumps( (name, timestamp, size, content_type, etag, deleted), protocol=PICKLE_PROTOCOL).encode('base64')) fp.flush() def is_deleted(self, timestamp=None): """ Check if the DB is considered to be deleted. :returns: True if the DB is considered to be deleted, False otherwise """ if self.db_file != ':memory:' and not os.path.exists(self.db_file): return True self._commit_puts_stale_ok() with self.get() as conn: row = conn.execute(''' SELECT put_timestamp, delete_timestamp, object_count FROM container_stat''').fetchone() # leave this db as a tombstone for a consistency window if timestamp and row['delete_timestamp'] > timestamp: return False # The container is considered deleted if the delete_timestamp # value is greater than the put_timestamp, and there are no # objects in the container. return (row['object_count'] in (None, '', 0, '0')) and \ (float(row['delete_timestamp']) > float(row['put_timestamp'])) def get_info(self): """ Get global data for the container. :returns: dict with keys: account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id, x_container_sync_point1, and x_container_sync_point2. """ self._commit_puts_stale_ok() with self.get() as conn: data = None trailing = 'x_container_sync_point1, x_container_sync_point2' while not data: try: data = conn.execute(''' SELECT account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id, %s FROM container_stat ''' % (trailing,)).fetchone() except sqlite3.OperationalError as err: if 'no such column: x_container_sync_point' in str(err): trailing = '-1 AS x_container_sync_point1, ' \ '-1 AS x_container_sync_point2' else: raise data = dict(data) return data def set_x_container_sync_points(self, sync_point1, sync_point2): with self.get() as conn: orig_isolation_level = conn.isolation_level try: # We turn off auto-transactions to ensure the alter table # commands are part of the transaction. conn.isolation_level = None conn.execute('BEGIN') try: self._set_x_container_sync_points(conn, sync_point1, sync_point2) except sqlite3.OperationalError as err: if 'no such column: x_container_sync_point' not in \ str(err): raise conn.execute(''' ALTER TABLE container_stat ADD COLUMN x_container_sync_point1 INTEGER DEFAULT -1 ''') conn.execute(''' ALTER TABLE container_stat ADD COLUMN x_container_sync_point2 INTEGER DEFAULT -1 ''') self._set_x_container_sync_points(conn, sync_point1, sync_point2) conn.execute('COMMIT') finally: conn.isolation_level = orig_isolation_level def _set_x_container_sync_points(self, conn, sync_point1, sync_point2): if sync_point1 is not None and sync_point2 is not None: conn.execute(''' UPDATE container_stat SET x_container_sync_point1 = ?, x_container_sync_point2 = ? ''', (sync_point1, sync_point2)) elif sync_point1 is not None: conn.execute(''' UPDATE container_stat SET x_container_sync_point1 = ? ''', (sync_point1,)) elif sync_point2 is not None: conn.execute(''' UPDATE container_stat SET x_container_sync_point2 = ? ''', (sync_point2,)) def reported(self, put_timestamp, delete_timestamp, object_count, bytes_used): """ Update reported stats, available with container's `get_info`. :param put_timestamp: put_timestamp to update :param delete_timestamp: delete_timestamp to update :param object_count: object_count to update :param bytes_used: bytes_used to update """ with self.get() as conn: conn.execute(''' UPDATE container_stat SET reported_put_timestamp = ?, reported_delete_timestamp = ?, reported_object_count = ?, reported_bytes_used = ? ''', (put_timestamp, delete_timestamp, object_count, bytes_used)) conn.commit() def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimter based on the path :returns: list of tuples of (name, created_at, size, content_type, etag) """ delim_force_gte = False (marker, end_marker, prefix, delimiter, path) = utf8encode( marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() if path is not None: prefix = path if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' elif delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if path is not None: if name == path: continue if end >= 0 and len(name) > end + len(delimiter): marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end > 0: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusinve of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '']) curs.close() break results.append(row) if not rowcount: break return results def merge_items(self, item_list, source=None): """ Merge items into the object table. :param item_list: list of dictionaries of {'name', 'created_at', 'size', 'content_type', 'etag', 'deleted'} :param source: if defined, update incoming_sync with the source """ with self.get() as conn: max_rowid = -1 for rec in item_list: query = ''' DELETE FROM object WHERE name = ? AND (created_at < ?) ''' if self.get_db_version(conn) >= 1: query += ' AND deleted IN (0, 1)' conn.execute(query, (rec['name'], rec['created_at'])) query = 'SELECT 1 FROM object WHERE name = ?' if self.get_db_version(conn) >= 1: query += ' AND deleted IN (0, 1)' if not conn.execute(query, (rec['name'],)).fetchall(): conn.execute(''' INSERT INTO object (name, created_at, size, content_type, etag, deleted) VALUES (?, ?, ?, ?, ?, ?) ''', ([rec['name'], rec['created_at'], rec['size'], rec['content_type'], rec['etag'], rec['deleted']])) if source: max_rowid = max(max_rowid, rec['ROWID']) if source: try: conn.execute(''' INSERT INTO incoming_sync (sync_point, remote_id) VALUES (?, ?) ''', (max_rowid, source)) except sqlite3.IntegrityError: conn.execute(''' UPDATE incoming_sync SET sync_point=max(?, sync_point) WHERE remote_id=? ''', (max_rowid, source)) conn.commit() swift-1.13.1/swift/container/__init__.py0000664000175400017540000000000012323703611021302 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/proxy/0000775000175400017540000000000012323703665016413 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/proxy/server.py0000664000175400017540000006057512323703614020302 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import mimetypes import os import socket from swift import gettext_ as _ from random import shuffle from time import time import itertools from eventlet import Timeout from swift import __canonical_version__ as swift_version from swift.common import constraints from swift.common.ring import Ring from swift.common.utils import cache_from_env, get_logger, \ get_remote_client, split_path, config_true_value, generate_trans_id, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ register_swift_info from swift.common.constraints import check_utf8 from swift.proxy.controllers import AccountController, ObjectController, \ ContainerController, InfoController from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPMethodNotAllowed, HTTPNotFound, HTTPPreconditionFailed, \ HTTPServerError, HTTPException, Request # List of entry points for mandatory middlewares. # # Fields: # # "name" (required) is the entry point name from setup.py. # # "after_fn" (optional) a function that takes a PipelineWrapper object as its # single argument and returns a list of middlewares that this middleware # should come after. Any middlewares in the returned list that are not present # in the pipeline will be ignored, so you can safely name optional middlewares # to come after. For example, ["catch_errors", "bulk"] would install this # middleware after catch_errors and bulk if both were present, but if bulk # were absent, would just install it after catch_errors. required_filters = [ {'name': 'catch_errors'}, {'name': 'gatekeeper', 'after_fn': lambda pipe: (['catch_errors'] if pipe.startswith("catch_errors") else [])}, {'name': 'dlo', 'after_fn': lambda _junk: ['catch_errors', 'gatekeeper', 'proxy_logging']}] class Application(object): """WSGI application for the proxy server.""" def __init__(self, conf, memcache=None, logger=None, account_ring=None, container_ring=None, object_ring=None): if conf is None: conf = {} if logger is None: self.logger = get_logger(conf, log_route='proxy-server') else: self.logger = logger swift_dir = conf.get('swift_dir', '/etc/swift') self.node_timeout = int(conf.get('node_timeout', 10)) self.recoverable_node_timeout = int( conf.get('recoverable_node_timeout', self.node_timeout)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) self.trans_id_suffix = conf.get('trans_id_suffix', '') self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5)) self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ int(conf.get('error_suppression_limit', 10)) self.recheck_container_existence = \ int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ config_true_value(conf.get('allow_account_management', 'no')) self.object_post_as_copy = \ config_true_value(conf.get('object_post_as_copy', 'true')) self.object_ring = object_ring or Ring(swift_dir, ring_name='object') self.container_ring = container_ring or Ring(swift_dir, ring_name='container') self.account_ring = account_ring or Ring(swift_dir, ring_name='account') self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) self.max_containers_per_account = \ int(conf.get('max_containers_per_account') or 0) self.max_containers_whitelist = [ a.strip() for a in conf.get('max_containers_whitelist', '').split(',') if a.strip()] self.deny_host_headers = [ host.strip() for host in conf.get('deny_host_headers', '').split(',') if host.strip()] self.rate_limit_after_segment = \ int(conf.get('rate_limit_after_segment', 10)) self.rate_limit_segments_per_sec = \ int(conf.get('rate_limit_segments_per_sec', 1)) self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true')) self.cors_allow_origin = [ a.strip() for a in conf.get('cors_allow_origin', '').split(',') if a.strip()] self.strict_cors_mode = config_true_value( conf.get('strict_cors_mode', 't')) self.node_timings = {} self.timing_expiry = int(conf.get('timing_expiry', 300)) self.sorting_method = conf.get('sorting_method', 'shuffle').lower() self.max_large_object_get_time = float( conf.get('max_large_object_get_time', '86400')) value = conf.get('request_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.request_node_count = lambda replicas: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.request_node_count = lambda replicas: value * replicas else: raise ValueError( 'Invalid request_node_count value: %r' % ''.join(value)) try: self._read_affinity = read_affinity = conf.get('read_affinity', '') self.read_affinity_sort_key = affinity_key_function(read_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid read_affinity value: %r (%s)" % (read_affinity, err.message)) try: write_affinity = conf.get('write_affinity', '') self.write_affinity_is_local_fn \ = affinity_locality_predicate(write_affinity) except ValueError as err: # make the message a little more useful raise ValueError("Invalid write_affinity value: %r (%s)" % (write_affinity, err.message)) value = conf.get('write_affinity_node_count', '2 * replicas').lower().split() if len(value) == 1: value = int(value[0]) self.write_affinity_node_count = lambda replicas: value elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas': value = int(value[0]) self.write_affinity_node_count = lambda replicas: value * replicas else: raise ValueError( 'Invalid write_affinity_node_count value: %r' % ''.join(value)) # swift_owner_headers are stripped by the account and container # controllers; we should extend header stripping to object controller # when a privileged object header is implemented. swift_owner_headers = conf.get( 'swift_owner_headers', 'x-container-read, x-container-write, ' 'x-container-sync-key, x-container-sync-to, ' 'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, ' 'x-account-access-control') self.swift_owner_headers = [ name.strip().title() for name in swift_owner_headers.split(',') if name.strip()] # Initialization was successful, so now apply the client chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because it affects the client as well, currently, we use the # client chunk size as the govenor and not the object chunk size. socket._fileobject.default_bufsize = self.client_chunk_size self.expose_info = config_true_value( conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( conf.get('disallowed_sections')) self.admin_key = conf.get('admin_key', None) register_swift_info( version=swift_version, max_file_size=constraints.MAX_FILE_SIZE, max_meta_name_length=constraints.MAX_META_NAME_LENGTH, max_meta_value_length=constraints.MAX_META_VALUE_LENGTH, max_meta_count=constraints.MAX_META_COUNT, account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT, container_listing_limit=constraints.CONTAINER_LISTING_LIMIT, max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH, max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH, max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH, strict_cors_mode=self.strict_cors_mode) def check_config(self): """ Check the configuration for possible errors """ if self._read_affinity and self.sorting_method != 'affinity': self.logger.warn("sorting_method is set to '%s', not 'affinity'; " "read_affinity setting will have no effect." % self.sorting_method) def get_controller(self, path): """ Get the controller to handle a request. :param path: path from request :returns: tuple of (controller class, path dictionary) :raises: ValueError (thrown by split_path) if given invalid path """ if path == '/info': d = dict(version=None, expose_info=self.expose_info, disallowed_sections=self.disallowed_sections, admin_key=self.admin_key) return InfoController, d version, account, container, obj = split_path(path, 1, 4, True) d = dict(version=version, account_name=account, container_name=container, object_name=obj) if obj and container and account: return ObjectController, d elif container and account: return ContainerController, d elif account and not container and not obj: return AccountController, d return None, d def __call__(self, env, start_response): """ WSGI entry point. Wraps env in swob.Request object and passes it down. :param env: WSGI environment dictionary :param start_response: WSGI callable """ try: if self.memcache is None: self.memcache = cache_from_env(env) req = self.update_request(Request(env)) return self.handle_request(req)(env, start_response) except UnicodeError: err = HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') return err(env, start_response) except (Exception, Timeout): start_response('500 Server Error', [('Content-Type', 'text/plain')]) return ['Internal server error.\n'] def update_request(self, req): if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token'] return req def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: controller, path_parts = self.get_controller(req.path) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id = generate_trans_id(self.trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp: # No resp means authorized, no delayed recheck required. del req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method return handler(req) except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req) def sort_nodes(self, nodes): ''' Sorts nodes in-place (and returns the sorted list) according to the configured strategy. The default "sorting" is to randomly shuffle the nodes. If the "timing" strategy is chosen, the nodes are sorted according to the stored timing data. ''' # In the case of timing sorting, shuffling ensures that close timings # (ie within the rounding resolution) won't prefer one over another. # Python's sort is stable (http://wiki.python.org/moin/HowTo/Sorting/) shuffle(nodes) if self.sorting_method == 'timing': now = time() def key_func(node): timing, expires = self.node_timings.get(node['ip'], (-1.0, 0)) return timing if expires > now else -1.0 nodes.sort(key=key_func) elif self.sorting_method == 'affinity': nodes.sort(key=self.read_affinity_sort_key) return nodes def set_node_timing(self, node, timing): if self.sorting_method != 'timing': return now = time() timing = round(timing, 3) # sort timings to the millisecond self.node_timings[node['ip']] = (timing, now + self.timing_expiry) def error_limited(self, node): """ Check if the node is currently error limited. :param node: dictionary of node to check :returns: True if error limited, False otherwise """ now = time() if 'errors' not in node: return False if 'last_error' in node and node['last_error'] < \ now - self.error_suppression_interval: del node['last_error'] if 'errors' in node: del node['errors'] return False limited = node['errors'] > self.error_suppression_limit if limited: self.logger.debug( _('Node error limited %(ip)s:%(port)s (%(device)s)'), node) return limited def error_limit(self, node, msg): """ Mark a node as error limited. This immediately pretends the node received enough errors to trigger error suppression. Use this for errors like Insufficient Storage. For other errors use :func:`error_occurred`. :param node: dictionary of node to error limit :param msg: error message """ node['errors'] = self.error_suppression_limit + 1 node['last_error'] = time() self.logger.error(_('%(msg)s %(ip)s:%(port)s/%(device)s'), {'msg': msg, 'ip': node['ip'], 'port': node['port'], 'device': node['device']}) def error_occurred(self, node, msg): """ Handle logging, and handling of errors. :param node: dictionary of node to handle errors for :param msg: error message """ node['errors'] = node.get('errors', 0) + 1 node['last_error'] = time() self.logger.error(_('%(msg)s %(ip)s:%(port)s/%(device)s'), {'msg': msg, 'ip': node['ip'], 'port': node['port'], 'device': node['device']}) def iter_nodes(self, ring, partition, node_iter=None): """ Yields nodes for a ring partition, skipping over error limited nodes and stopping at the configurable number of nodes. If a node yielded subsequently gets error limited, an extra node will be yielded to take its place. Note that if you're going to iterate over this concurrently from multiple greenthreads, you'll want to use a swift.common.utils.GreenthreadSafeIterator to serialize access. Otherwise, you may get ValueErrors from concurrent access. (You also may not, depending on how logging is configured, the vagaries of socket IO and eventlet, and the phase of the moon.) :param ring: ring to get yield nodes from :param partition: ring partition to yield nodes for :param node_iter: optional iterable of nodes to try. Useful if you want to filter or reorder the nodes. """ part_nodes = ring.get_part_nodes(partition) if node_iter is None: node_iter = itertools.chain(part_nodes, ring.get_more_nodes(partition)) num_primary_nodes = len(part_nodes) # Use of list() here forcibly yanks the first N nodes (the primary # nodes) from node_iter, so the rest of its values are handoffs. primary_nodes = self.sort_nodes( list(itertools.islice(node_iter, num_primary_nodes))) handoff_nodes = node_iter nodes_left = self.request_node_count(len(primary_nodes)) for node in primary_nodes: if not self.error_limited(node): yield node if not self.error_limited(node): nodes_left -= 1 if nodes_left <= 0: return handoffs = 0 for node in handoff_nodes: if not self.error_limited(node): handoffs += 1 if self.log_handoffs: self.logger.increment('handoff_count') self.logger.warning( 'Handoff requested (%d)' % handoffs) if handoffs == len(primary_nodes): self.logger.increment('handoff_all_count') yield node if not self.error_limited(node): nodes_left -= 1 if nodes_left <= 0: return def exception_occurred(self, node, typ, additional_info): """ Handle logging of generic exceptions. :param node: dictionary of node to log the error for :param typ: server type :param additional_info: additional information to log """ self.logger.exception( _('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: ' '%(info)s'), {'type': typ, 'ip': node['ip'], 'port': node['port'], 'device': node['device'], 'info': additional_info}) def modify_wsgi_pipeline(self, pipe): """ Called during WSGI pipeline creation. Modifies the WSGI pipeline context to ensure that mandatory middleware is present in the pipeline. :param pipe: A PipelineWrapper object """ pipeline_was_modified = False for filter_spec in reversed(required_filters): filter_name = filter_spec['name'] if filter_name not in pipe: afters = filter_spec.get('after_fn', lambda _junk: [])(pipe) insert_at = 0 for after in afters: try: insert_at = max(insert_at, pipe.index(after) + 1) except ValueError: # not in pipeline; ignore it pass self.logger.info( 'Adding required filter %s to pipeline at position %d' % (filter_name, insert_at)) ctx = pipe.create_filter(filter_name) pipe.insert_filter(ctx, index=insert_at) pipeline_was_modified = True if pipeline_was_modified: self.logger.info("Pipeline was modified. New pipeline is \"%s\".", pipe) else: self.logger.debug("Pipeline is \"%s\"", pipe) def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI proxy apps.""" conf = global_conf.copy() conf.update(local_conf) app = Application(conf) app.check_config() return app swift-1.13.1/swift/proxy/controllers/0000775000175400017540000000000012323703665020761 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/proxy/controllers/obj.py0000664000175400017540000011753012323703614022106 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # NOTE: swift_conn # You'll see swift_conn passed around a few places in this file. This is the # source bufferedhttp connection of whatever it is attached to. # It is used when early termination of reading from the connection should # happen, such as when a range request is satisfied but there's still more the # source connection would like to send. To prevent having to read all the data # that could be left, the source connection can be .close() and then reads # commence to empty out any buffers. # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. import itertools import mimetypes import time import math from swift import gettext_ as _ from urllib import unquote, quote from eventlet import GreenPile from eventlet.queue import Queue from eventlet.timeout import Timeout from swift.common.utils import ContextPool, normalize_timestamp, \ config_true_value, public, json, csv_append, GreenthreadSafeIterator, \ quorum_size, GreenAsyncPile, normalize_delete_at_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ MAX_FILE_SIZE, check_copy_from_header from swift.common.exceptions import ChunkReadTimeout, \ ChunkWriteTimeout, ConnectionTimeout, ListingIterNotFound, \ ListingIterNotAuthorized, ListingIterError from swift.common.http import is_success, is_client_error, HTTP_CONTINUE, \ HTTP_CREATED, HTTP_MULTIPLE_CHOICES, HTTP_NOT_FOUND, \ HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE, \ HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPNotFound, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPRequestTimeout, \ HTTPServerError, HTTPServiceUnavailable, Request, \ HTTPClientDisconnect, HTTPNotImplemented from swift.common.request_helpers import is_user_meta def copy_headers_into(from_r, to_r): """ Will copy desired headers from from_r to to_r :params from_r: a swob Request or Response :params to_r: a swob Request or Response """ pass_headers = ['x-delete-at'] for k, v in from_r.headers.items(): if is_user_meta('object', k) or k.lower() in pass_headers: to_r.headers[k] = v def check_content_type(req): if not req.environ.get('swift.content_type_overridden') and \ ';' in req.headers.get('content-type', ''): for param in req.headers['content-type'].split(';')[1:]: if param.lstrip().startswith('swift_'): return HTTPBadRequest("Invalid Content-Type, " "swift_* is not a valid parameter name.") return None class ObjectController(Controller): """WSGI controller for object requests.""" server_type = 'Object' def __init__(self, app, account_name, container_name, object_name, **kwargs): Controller.__init__(self, app) self.account_name = unquote(account_name) self.container_name = unquote(container_name) self.object_name = unquote(object_name) def _listing_iter(self, lcontainer, lprefix, env): for page in self._listing_pages_iter(lcontainer, lprefix, env): for item in page: yield item def _listing_pages_iter(self, lcontainer, lprefix, env): lpartition = self.app.container_ring.get_part( self.account_name, lcontainer) marker = '' while True: lreq = Request.blank('i will be overridden by env', environ=env) # Don't quote PATH_INFO, by WSGI spec lreq.environ['PATH_INFO'] = \ '/v1/%s/%s' % (self.account_name, lcontainer) lreq.environ['REQUEST_METHOD'] = 'GET' lreq.environ['QUERY_STRING'] = \ 'format=json&prefix=%s&marker=%s' % (quote(lprefix), quote(marker)) lresp = self.GETorHEAD_base( lreq, _('Container'), self.app.container_ring, lpartition, lreq.swift_entity_path) if 'swift.authorize' in env: lreq.acl = lresp.headers.get('x-container-read') aresp = env['swift.authorize'](lreq) if aresp: raise ListingIterNotAuthorized(aresp) if lresp.status_int == HTTP_NOT_FOUND: raise ListingIterNotFound() elif not is_success(lresp.status_int): raise ListingIterError() if not lresp.body: break sublisting = json.loads(lresp.body) if not sublisting: break marker = sublisting[-1]['name'].encode('utf-8') yield sublisting def _remaining_items(self, listing_iter): """ Returns an item-by-item iterator for a page-by-page iterator of item listings. Swallows listing-related errors; this iterator is only used after we've already started streaming a response to the client, and so if we start getting errors from the container servers now, it's too late to send an error to the client, so we just quit looking for segments. """ try: for page in listing_iter: for item in page: yield item except ListingIterNotFound: pass except ListingIterError: pass except ListingIterNotAuthorized: pass def iter_nodes_local_first(self, ring, partition): """ Yields nodes for a ring partition. If the 'write_affinity' setting is non-empty, then this will yield N local nodes (as defined by the write_affinity setting) first, then the rest of the nodes as normal. It is a re-ordering of the nodes such that the local ones come first; no node is omitted. The effect is that the request will be serviced by local object servers first, but nonlocal ones will be employed if not enough local ones are available. :param ring: ring to get nodes from :param partition: ring partition to yield nodes for """ primary_nodes = ring.get_part_nodes(partition) num_locals = self.app.write_affinity_node_count(len(primary_nodes)) is_local = self.app.write_affinity_is_local_fn if is_local is None: return self.app.iter_nodes(ring, partition) all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition)) first_n_local_nodes = list(itertools.islice( itertools.ifilter(is_local, all_nodes), num_locals)) # refresh it; it moved when we computed first_n_local_nodes all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition)) local_first_node_iter = itertools.chain( first_n_local_nodes, itertools.ifilter(lambda node: node not in first_n_local_nodes, all_nodes)) return self.app.iter_nodes( ring, partition, node_iter=local_first_node_iter) def GETorHEAD(self, req): """Handle HTTP GET or HEAD requests.""" container_info = self.container_info( self.account_name, self.container_name, req) req.acl = container_info['read_acl'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp partition = self.app.object_ring.get_part( self.account_name, self.container_name, self.object_name) resp = self.GETorHEAD_base( req, _('Object'), self.app.object_ring, partition, req.swift_entity_path) if ';' in resp.headers.get('content-type', ''): # strip off swift_bytes from content-type content_type, check_extra_meta = \ resp.headers['content-type'].rsplit(';', 1) if check_extra_meta.lstrip().startswith('swift_bytes='): resp.content_type = content_type return resp @public @cors_validation @delay_denial def GET(self, req): """Handler for HTTP GET requests.""" return self.GETorHEAD(req) @public @cors_validation @delay_denial def HEAD(self, req): """Handler for HTTP HEAD requests.""" return self.GETorHEAD(req) @public @cors_validation @delay_denial def POST(self, req): """HTTP POST request handler.""" if 'x-delete-after' in req.headers: try: x_delete_after = int(req.headers['x-delete-after']) except ValueError: return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-After') req.headers['x-delete-at'] = normalize_delete_at_timestamp( time.time() + x_delete_after) if self.app.object_post_as_copy: req.method = 'PUT' req.path_info = '/v1/%s/%s/%s' % ( self.account_name, self.container_name, self.object_name) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, self.object_name)) req.headers['X-Fresh-Metadata'] = 'true' req.environ['swift_versioned_copy'] = True if req.environ.get('QUERY_STRING'): req.environ['QUERY_STRING'] += '&multipart-manifest=get' else: req.environ['QUERY_STRING'] = 'multipart-manifest=get' resp = self.PUT(req) # Older editions returned 202 Accepted on object POSTs, so we'll # convert any 201 Created responses to that for compatibility with # picky clients. if resp.status_int != HTTP_CREATED: return resp return HTTPAccepted(request=req) else: error_response = check_metadata(req, 'object') if error_response: return error_response container_info = self.container_info( self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) if 'x-delete-at' in req.headers: try: x_delete_at = normalize_delete_at_timestamp( int(req.headers['x-delete-at'])) if int(x_delete_at) < time.time(): return HTTPBadRequest( body='X-Delete-At in past', request=req, content_type='text/plain') except ValueError: return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-At') req.environ.setdefault('swift.log_info', []).append( 'x-delete-at:%s' % x_delete_at) delete_at_container = normalize_delete_at_timestamp( int(x_delete_at) / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) delete_at_part, delete_at_nodes = \ self.app.container_ring.get_nodes( self.app.expiring_objects_account, delete_at_container) else: delete_at_container = delete_at_part = delete_at_nodes = None partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) req.headers['X-Timestamp'] = normalize_timestamp(time.time()) headers = self._backend_requests( req, len(nodes), container_partition, containers, delete_at_container, delete_at_part, delete_at_nodes) resp = self.make_requests(req, self.app.object_ring, partition, 'POST', req.swift_entity_path, headers) return resp def _backend_requests(self, req, n_outgoing, container_partition, containers, delete_at_container=None, delete_at_partition=None, delete_at_nodes=None): headers = [self.generate_request_headers(req, additional=req.headers) for _junk in range(n_outgoing)] for header in headers: header['Connection'] = 'close' for i, container in enumerate(containers): i = i % len(headers) headers[i]['X-Container-Partition'] = container_partition headers[i]['X-Container-Host'] = csv_append( headers[i].get('X-Container-Host'), '%(ip)s:%(port)s' % container) headers[i]['X-Container-Device'] = csv_append( headers[i].get('X-Container-Device'), container['device']) for i, node in enumerate(delete_at_nodes or []): i = i % len(headers) headers[i]['X-Delete-At-Container'] = delete_at_container headers[i]['X-Delete-At-Partition'] = delete_at_partition headers[i]['X-Delete-At-Host'] = csv_append( headers[i].get('X-Delete-At-Host'), '%(ip)s:%(port)s' % node) headers[i]['X-Delete-At-Device'] = csv_append( headers[i].get('X-Delete-At-Device'), node['device']) return headers def _send_file(self, conn, path): """Method for a file PUT coro""" while True: chunk = conn.queue.get() if not conn.failed: try: with ChunkWriteTimeout(self.app.node_timeout): conn.send(chunk) except (Exception, ChunkWriteTimeout): conn.failed = True self.app.exception_occurred( conn.node, _('Object'), _('Trying to write to %s') % path) conn.queue.task_done() def _connect_put_node(self, nodes, part, path, headers, logger_thread_locals): """Method for a file PUT connect""" self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: start_time = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect( node['ip'], node['port'], node['device'], part, 'PUT', path, headers) self.app.set_node_timing(node, time.time() - start_time) with Timeout(self.app.node_timeout): resp = conn.getexpect() if resp.status == HTTP_CONTINUE: conn.resp = None conn.node = node return conn elif is_success(resp.status): conn.resp = resp conn.node = node return conn elif headers['If-None-Match'] is not None and \ resp.status == HTTP_PRECONDITION_FAILED: conn.resp = resp conn.node = node return conn elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) except (Exception, Timeout): self.app.exception_occurred( node, _('Object'), _('Expect: 100-continue on %s') % path) def _get_put_responses(self, req, conns, nodes): statuses = [] reasons = [] bodies = [] etags = set() def get_conn_response(conn): try: with Timeout(self.app.node_timeout): if conn.resp: return conn.resp else: return conn.getresponse() except (Exception, Timeout): self.app.exception_occurred( conn.node, _('Object'), _('Trying to get final status of PUT to %s') % req.path) pile = GreenAsyncPile(len(conns)) for conn in conns: pile.spawn(get_conn_response, conn) for response in pile: if response: statuses.append(response.status) reasons.append(response.reason) bodies.append(response.read()) if response.status >= HTTP_INTERNAL_SERVER_ERROR: self.app.error_occurred( conn.node, _('ERROR %(status)d %(body)s From Object Server ' 're: %(path)s') % {'status': response.status, 'body': bodies[-1][:1024], 'path': req.path}) elif is_success(response.status): etags.add(response.getheader('etag').strip('"')) if self.have_quorum(statuses, len(nodes)): break # give any pending requests *some* chance to finish pile.waitall(self.app.post_quorum_timeout) while len(statuses) < len(nodes): statuses.append(HTTP_SERVICE_UNAVAILABLE) reasons.append('') bodies.append('') return statuses, reasons, bodies, etags @public @cors_validation @delay_denial def PUT(self, req): """HTTP PUT request handler.""" if req.if_none_match is not None and '*' not in req.if_none_match: # Sending an etag with if-none-match isn't currently supported return HTTPBadRequest(request=req, content_type='text/plain', body='If-None-Match only supports *') container_info = self.container_info( self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) try: ml = req.message_length() except ValueError as e: return HTTPBadRequest(request=req, content_type='text/plain', body=str(e)) except AttributeError as e: return HTTPNotImplemented(request=req, content_type='text/plain', body=str(e)) if ml is not None and ml > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) if 'x-delete-after' in req.headers: try: x_delete_after = int(req.headers['x-delete-after']) except ValueError: return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-After') req.headers['x-delete-at'] = normalize_delete_at_timestamp( time.time() + x_delete_after) partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) # do a HEAD request for container sync and checking object versions if 'x-timestamp' in req.headers or \ (object_versions and not req.environ.get('swift_versioned_copy')): hreq = Request.blank(req.path_info, headers={'X-Newest': 'True'}, environ={'REQUEST_METHOD': 'HEAD'}) hresp = self.GETorHEAD_base( hreq, _('Object'), self.app.object_ring, partition, hreq.swift_entity_path) # Used by container sync feature if 'x-timestamp' in req.headers: try: req.headers['X-Timestamp'] = \ normalize_timestamp(req.headers['x-timestamp']) if hresp.environ and 'swift_x_timestamp' in hresp.environ and \ float(hresp.environ['swift_x_timestamp']) >= \ float(req.headers['x-timestamp']): return HTTPAccepted(request=req) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) else: req.headers['X-Timestamp'] = normalize_timestamp(time.time()) # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): guessed_type, _junk = mimetypes.guess_type(req.path_info) req.headers['Content-Type'] = guessed_type or \ 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') else: content_type_manually_set = False error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response if object_versions and not req.environ.get('swift_versioned_copy'): if hresp.status_int != HTTP_NOT_FOUND: # This is a version manifest and needs to be handled # differently. First copy the existing data to a new object, # then write the data from this request to the version manifest # object. lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' ts_source = hresp.environ.get('swift_x_timestamp') if ts_source is None: ts_source = time.mktime(time.strptime( hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) new_ts = normalize_timestamp(ts_source) vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} copy_environ = {'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } copy_req = Request.blank(req.path_info, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(copy_req) if is_client_error(copy_resp.status_int): # missing container or bad permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) reader = req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % source_header) src_container_name, src_obj_name = check_copy_from_header(req) ver, acct, _rest = req.split_path(2, 3, True) if isinstance(acct, unicode): acct = acct.encode('utf-8') source_header = '/%s/%s/%s/%s' % (ver, acct, src_container_name, src_obj_name) source_req = req.copy_get() source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name self.object_name = src_obj_name self.container_name = src_container_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp self.object_name = orig_obj_name self.container_name = orig_container_name data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if sink_req.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. return HTTPRequestEntityTooLarge(request=req) if sink_req.content_length > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if not content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if not config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req if 'x-delete-at' in req.headers: try: x_delete_at = normalize_delete_at_timestamp( int(req.headers['x-delete-at'])) if int(x_delete_at) < time.time(): return HTTPBadRequest( body='X-Delete-At in past', request=req, content_type='text/plain') except ValueError: return HTTPBadRequest(request=req, content_type='text/plain', body='Non-integer X-Delete-At') req.environ.setdefault('swift.log_info', []).append( 'x-delete-at:%s' % x_delete_at) delete_at_container = normalize_delete_at_timestamp( int(x_delete_at) / self.app.expiring_objects_container_divisor * self.app.expiring_objects_container_divisor) delete_at_part, delete_at_nodes = \ self.app.container_ring.get_nodes( self.app.expiring_objects_account, delete_at_container) else: delete_at_container = delete_at_part = delete_at_nodes = None node_iter = GreenthreadSafeIterator( self.iter_nodes_local_first(self.app.object_ring, partition)) pile = GreenPile(len(nodes)) te = req.headers.get('transfer-encoding', '') chunked = ('chunked' in te) outgoing_headers = self._backend_requests( req, len(nodes), container_partition, containers, delete_at_container, delete_at_part, delete_at_nodes) for nheaders in outgoing_headers: # RFC2616:8.2.3 disallows 100-continue without a body if (req.content_length > 0) or chunked: nheaders['Expect'] = '100-continue' pile.spawn(self._connect_put_node, node_iter, partition, req.swift_entity_path, nheaders, self.app.logger.thread_locals) conns = [conn for conn in pile if conn] min_conns = quorum_size(len(nodes)) if req.if_none_match is not None and '*' in req.if_none_match: statuses = [conn.resp.status for conn in conns if conn.resp] if HTTP_PRECONDITION_FAILED in statuses: # If we find any copy of the file, it shouldn't be uploaded self.app.logger.debug( _('Object PUT returning 412, %(statuses)r'), {'statuses': statuses}) return HTTPPreconditionFailed(request=req) if len(conns) < min_conns: self.app.logger.error( _('Object PUT returning 503, %(conns)s/%(nodes)s ' 'required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) bytes_transferred = 0 try: with ContextPool(len(nodes)) as pool: for conn in conns: conn.failed = False conn.queue = Queue(self.app.put_queue_depth) pool.spawn(self._send_file, conn, req.path) while True: with ChunkReadTimeout(self.app.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: for conn in conns: conn.queue.put('0\r\n\r\n') break bytes_transferred += len(chunk) if bytes_transferred > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) for conn in list(conns): if not conn.failed: conn.queue.put( '%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk) else: conns.remove(conn) if len(conns) < min_conns: self.app.logger.error(_( 'Object PUT exceptions during' ' send, %(conns)s/%(nodes)s required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) for conn in conns: if conn.queue.unfinished_tasks: conn.queue.join() conns = [conn for conn in conns if not conn.failed] except ChunkReadTimeout as err: self.app.logger.warn( _('ERROR Client read timeout (%ss)'), err.seconds) self.app.logger.increment('client_timeouts') return HTTPRequestTimeout(request=req) except (Exception, Timeout): self.app.logger.exception( _('ERROR Exception causing client disconnect')) return HTTPClientDisconnect(request=req) if req.content_length and bytes_transferred < req.content_length: req.client_disconnect = True self.app.logger.warn( _('Client disconnected without sending enough data')) self.app.logger.increment('client_disconnects') return HTTPClientDisconnect(request=req) statuses, reasons, bodies, etags = self._get_put_responses(req, conns, nodes) if len(etags) > 1: self.app.logger.error( _('Object servers returned %s mismatched etags'), len(etags)) return HTTPServerError(request=req) etag = etags.pop() if len(etags) else None resp = self.best_response(req, statuses, reasons, bodies, _('Object PUT'), etag=etag) if source_header: resp.headers['X-Copied-From'] = quote( source_header.split('/', 3)[3]) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) resp.last_modified = math.ceil(float(req.headers['X-Timestamp'])) return resp @public @cors_validation @delay_denial def DELETE(self, req): """HTTP DELETE request handler.""" container_info = self.container_info( self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if object_versions: # this is a version manifest and needs to be handled differently object_versions = unquote(object_versions) lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' last_item = None try: for last_item in self._listing_iter(lcontainer, lprefix, req.environ): pass except ListingIterNotFound: # no worries, last_item is None pass except ListingIterNotAuthorized as err: return err.aresp except ListingIterError: return HTTPServerError(request=req) if last_item: # there are older versions so copy the previous version to the # current object and delete the previous version orig_container = self.container_name orig_obj = self.object_name self.container_name = lcontainer self.object_name = last_item['name'].encode('utf-8') copy_path = '/v1/' + self.account_name + '/' + \ self.container_name + '/' + self.object_name copy_headers = {'X-Newest': 'True', 'Destination': orig_container + '/' + orig_obj } copy_environ = {'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } creq = Request.blank(copy_path, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(creq) if is_client_error(copy_resp.status_int): # some user error, maybe permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) # reset these because the COPY changed them self.container_name = lcontainer self.object_name = last_item['name'].encode('utf-8') new_del_req = Request.blank(copy_path, environ=req.environ) container_info = self.container_info( self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] new_del_req.acl = container_info['write_acl'] new_del_req.path_info = copy_path req = new_del_req # remove 'X-If-Delete-At', since it is not for the older copy if 'X-If-Delete-At' in req.headers: del req.headers['X-If-Delete-At'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) # Used by container sync feature if 'x-timestamp' in req.headers: try: req.headers['X-Timestamp'] = \ normalize_timestamp(req.headers['x-timestamp']) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) else: req.headers['X-Timestamp'] = normalize_timestamp(time.time()) headers = self._backend_requests( req, len(nodes), container_partition, containers) resp = self.make_requests(req, self.app.object_ring, partition, 'DELETE', req.swift_entity_path, headers) return resp @public @cors_validation @delay_denial def COPY(self, req): """HTTP COPY request handler.""" dest = req.headers.get('Destination') if not dest: return HTTPPreconditionFailed(request=req, body='Destination header required') dest = unquote(dest) if not dest.startswith('/'): dest = '/' + dest try: _junk, dest_container, dest_object = dest.split('/', 2) except ValueError: return HTTPPreconditionFailed( request=req, body='Destination header must be of the form ' '/') source = '/' + self.container_name + '/' + self.object_name self.container_name = dest_container self.object_name = dest_object # re-write the existing request as a PUT instead of creating a new one # since this one is already attached to the posthooklogger req.method = 'PUT' req.path_info = '/v1/' + self.account_name + dest req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] return self.PUT(req) swift-1.13.1/swift/proxy/controllers/account.py0000664000175400017540000001475612323703614022776 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift import gettext_ as _ from urllib import unquote from swift.account.utils import account_listing_response from swift.common.request_helpers import get_listing_content_type from swift.common.middleware.acl import parse_acl, format_acl from swift.common.utils import public from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH from swift.common.http import HTTP_NOT_FOUND, HTTP_GONE from swift.proxy.controllers.base import Controller, clear_info_cache from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed from swift.common.request_helpers import get_sys_meta_prefix class AccountController(Controller): """WSGI controller for account requests""" server_type = 'Account' def __init__(self, app, account_name, **kwargs): Controller.__init__(self, app) self.account_name = unquote(account_name) if not self.app.allow_account_management: self.allowed_methods.remove('PUT') self.allowed_methods.remove('DELETE') def add_acls_from_sys_metadata(self, resp): if resp.environ['REQUEST_METHOD'] in ('HEAD', 'GET', 'PUT', 'POST'): prefix = get_sys_meta_prefix('account') + 'core-' name = 'access-control' (extname, intname) = ('x-account-' + name, prefix + name) acl_dict = parse_acl(version=2, data=resp.headers.pop(intname)) if acl_dict: # treat empty dict as empty header resp.headers[extname] = format_acl( version=2, acl_dict=acl_dict) def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp partition, nodes = self.app.account_ring.get_nodes(self.account_name) resp = self.GETorHEAD_base( req, _('Account'), self.app.account_ring, partition, req.swift_entity_path.rstrip('/')) if resp.status_int == HTTP_NOT_FOUND: if resp.headers.get('X-Account-Status', '').lower() == 'deleted': resp.status = HTTP_GONE elif self.app.account_autocreate: resp = account_listing_response(self.account_name, req, get_listing_content_type(req)) if req.environ.get('swift_owner'): self.add_acls_from_sys_metadata(resp) else: for header in self.app.swift_owner_headers: resp.headers.pop(header, None) return resp @public def PUT(self, req): """HTTP PUT request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) error_response = check_metadata(req, 'account') if error_response: return error_response if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'PUT', req.swift_entity_path, [headers] * len(accounts)) self.add_acls_from_sys_metadata(resp) return resp @public def POST(self, req): """HTTP POST request handler.""" if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp error_response = check_metadata(req, 'account') if error_response: return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req, transfer=True) clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.swift_entity_path, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: self.autocreate_account(req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'POST', req.swift_entity_path, [headers] * len(accounts)) self.add_acls_from_sys_metadata(resp) return resp @public def DELETE(self, req): """HTTP DELETE request handler.""" # Extra safety in case someone typos a query string for an # account-level DELETE request that was really meant to be caught by # some middleware. if req.query_string: return HTTPBadRequest(request=req) if not self.app.allow_account_management: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(self.allowed_methods)}) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = self.generate_request_headers(req) clear_info_cache(self.app, req.environ, self.account_name) resp = self.make_requests( req, self.app.account_ring, account_partition, 'DELETE', req.swift_entity_path, [headers] * len(accounts)) return resp swift-1.13.1/swift/proxy/controllers/base.py0000664000175400017540000014376612323703611022255 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # NOTE: swift_conn # You'll see swift_conn passed around a few places in this file. This is the # source bufferedhttp connection of whatever it is attached to. # It is used when early termination of reading from the connection should # happen, such as when a range request is satisfied but there's still more the # source connection would like to send. To prevent having to read all the data # that could be left, the source connection can be .close() and then reads # commence to empty out any buffers. # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. import os import time import functools import inspect from sys import exc_info from swift import gettext_ as _ from urllib import quote from eventlet import sleep from eventlet.timeout import Timeout from swift.common.wsgi import make_pre_authed_env from swift.common.utils import normalize_timestamp, config_true_value, \ public, split_path, list_from_csv, GreenthreadSafeIterator, \ quorum_size, GreenAsyncPile from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \ ConnectionTimeout from swift.common.http import is_informational, is_success, is_redirection, \ is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \ HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \ HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED from swift.common.swob import Request, Response, HeaderKeyDict, Range, \ HTTPException, HTTPRequestedRangeNotSatisfiable from swift.common.request_helpers import strip_sys_meta_prefix, \ strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta def update_headers(response, headers): """ Helper function to update headers in the response. :param response: swob.Response object :param headers: dictionary headers """ if hasattr(headers, 'items'): headers = headers.items() for name, value in headers: if name == 'etag': response.headers[name] = value.replace('"', '') elif name not in ('date', 'content-length', 'content-type', 'connection', 'x-put-timestamp', 'x-delete-after'): response.headers[name] = value def source_key(resp): """ Provide the timestamp of the swift http response as a floating point value. Used as a sort key. :param resp: bufferedhttp response object """ return float(resp.getheader('x-put-timestamp') or resp.getheader('x-timestamp') or 0) def delay_denial(func): """ Decorator to declare which methods should have any swift.authorize call delayed. This is so the method can load the Request object up with additional information that may be needed by the authorization system. :param func: function for which authorization will be delayed """ func.delay_denial = True @functools.wraps(func) def wrapped(*a, **kw): return func(*a, **kw) return wrapped def get_account_memcache_key(account): cache_key, env_key = _get_cache_key(account, None) return cache_key def get_container_memcache_key(account, container): if not container: raise ValueError("container not provided") cache_key, env_key = _get_cache_key(account, container) return cache_key def _prep_headers_to_info(headers, server_type): """ Helper method that iterates once over a dict of headers, converting all keys to lower case and separating into subsets containing user metadata, system metadata and other headers. """ meta = {} sysmeta = {} other = {} for key, val in dict(headers).iteritems(): lkey = key.lower() if is_user_meta(server_type, lkey): meta[strip_user_meta_prefix(server_type, lkey)] = val elif is_sys_meta(server_type, lkey): sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val else: other[lkey] = val return other, meta, sysmeta def headers_to_account_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of account info based on response headers. """ headers, meta, sysmeta = _prep_headers_to_info(headers, 'account') return { 'status': status_int, # 'container_count' anomaly: # Previous code sometimes expects an int sometimes a string # Current code aligns to str and None, yet translates to int in # deprecated functions as needed 'container_count': headers.get('x-account-container-count'), 'total_object_count': headers.get('x-account-object-count'), 'bytes': headers.get('x-account-bytes-used'), 'meta': meta, 'sysmeta': sysmeta } def headers_to_container_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of container info based on response headers. """ headers, meta, sysmeta = _prep_headers_to_info(headers, 'container') return { 'status': status_int, 'read_acl': headers.get('x-container-read'), 'write_acl': headers.get('x-container-write'), 'sync_key': headers.get('x-container-sync-key'), 'object_count': headers.get('x-container-object-count'), 'bytes': headers.get('x-container-bytes-used'), 'versions': headers.get('x-versions-location'), 'cors': { 'allow_origin': meta.get('access-control-allow-origin'), 'expose_headers': meta.get('access-control-expose-headers'), 'max_age': meta.get('access-control-max-age') }, 'meta': meta, 'sysmeta': sysmeta } def headers_to_object_info(headers, status_int=HTTP_OK): """ Construct a cacheable dict of object info based on response headers. """ headers, meta, sysmeta = _prep_headers_to_info(headers, 'object') info = {'status': status_int, 'length': headers.get('content-length'), 'type': headers.get('content-type'), 'etag': headers.get('etag'), 'meta': meta } return info def cors_validation(func): """ Decorator to check if the request is a CORS request and if so, if it's valid. :param func: function to check """ @functools.wraps(func) def wrapped(*a, **kw): controller = a[0] req = a[1] # The logic here was interpreted from # http://www.w3.org/TR/cors/#resource-requests # Is this a CORS request? req_origin = req.headers.get('Origin', None) if req_origin: # Yes, this is a CORS request so test if the origin is allowed container_info = \ controller.container_info(controller.account_name, controller.container_name, req) cors_info = container_info.get('cors', {}) # Call through to the decorated method resp = func(*a, **kw) if controller.app.strict_cors_mode and \ not controller.is_origin_allowed(cors_info, req_origin): return resp # Expose, # - simple response headers, # http://www.w3.org/TR/cors/#simple-response-header # - swift specific: etag, x-timestamp, x-trans-id # - user metadata headers # - headers provided by the user in # x-container-meta-access-control-expose-headers if 'Access-Control-Expose-Headers' not in resp.headers: expose_headers = [ 'cache-control', 'content-language', 'content-type', 'expires', 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id'] for header in resp.headers: if header.startswith('X-Container-Meta') or \ header.startswith('X-Object-Meta'): expose_headers.append(header.lower()) if cors_info.get('expose_headers'): expose_headers.extend( [header_line.strip() for header_line in cors_info['expose_headers'].split(' ') if header_line.strip()]) resp.headers['Access-Control-Expose-Headers'] = \ ', '.join(expose_headers) # The user agent won't process the response if the Allow-Origin # header isn't included if 'Access-Control-Allow-Origin' not in resp.headers: if cors_info['allow_origin'] and \ cors_info['allow_origin'].strip() == '*': resp.headers['Access-Control-Allow-Origin'] = '*' else: resp.headers['Access-Control-Allow-Origin'] = req_origin return resp else: # Not a CORS request so make the call as normal return func(*a, **kw) return wrapped def get_object_info(env, app, path=None, swift_source=None): """ Get the info structure for an object, based on env and app. This is useful to middlewares. .. note:: This call bypasses auth. Success does not imply that the request has authorization to the object. """ (version, account, container, obj) = \ split_path(path or env['PATH_INFO'], 4, 4, True) info = _get_object_info(app, env, account, container, obj, swift_source=swift_source) if not info: info = headers_to_object_info({}, 0) return info def get_container_info(env, app, swift_source=None): """ Get the info structure for a container, based on env and app. This is useful to middlewares. .. note:: This call bypasses auth. Success does not imply that the request has authorization to the container. """ (version, account, container, unused) = \ split_path(env['PATH_INFO'], 3, 4, True) info = get_info(app, env, account, container, ret_not_found=True, swift_source=swift_source) if not info: info = headers_to_container_info({}, 0) return info def get_account_info(env, app, swift_source=None): """ Get the info structure for an account, based on env and app. This is useful to middlewares. .. note:: This call bypasses auth. Success does not imply that the request has authorization to the account. """ (version, account, _junk, _junk) = \ split_path(env['PATH_INFO'], 2, 4, True) info = get_info(app, env, account, ret_not_found=True, swift_source=swift_source) if not info: info = headers_to_account_info({}, 0) if info.get('container_count') is None: info['container_count'] = 0 else: info['container_count'] = int(info['container_count']) return info def _get_cache_key(account, container): """ Get the keys for both memcache (cache_key) and env (env_key) where info about accounts and containers is cached :param account: The name of the account :param container: The name of the container (or None if account) :returns a tuple of (cache_key, env_key) """ if container: cache_key = 'container/%s/%s' % (account, container) else: cache_key = 'account/%s' % account # Use a unique environment cache key per account and one container. # This allows caching both account and container and ensures that when we # copy this env to form a new request, it won't accidentally reuse the # old container or account info env_key = 'swift.%s' % cache_key return cache_key, env_key def get_object_env_key(account, container, obj): """ Get the keys for env (env_key) where info about object is cached :param account: The name of the account :param container: The name of the container :param obj: The name of the object :returns a string env_key """ env_key = 'swift.object/%s/%s/%s' % (account, container, obj) return env_key def _set_info_cache(app, env, account, container, resp): """ Cache info in both memcache and env. Caching is used to avoid unnecessary calls to account & container servers. This is a private function that is being called by GETorHEAD_base and by clear_info_cache. Any attempt to GET or HEAD from the container/account server should use the GETorHEAD_base interface which would than set the cache. :param app: the application object :param account: the unquoted account name :param container: the unquoted container name or None :param resp: the response received or None if info cache should be cleared """ if container: cache_time = app.recheck_container_existence else: cache_time = app.recheck_account_existence cache_key, env_key = _get_cache_key(account, container) if resp: if resp.status_int == HTTP_NOT_FOUND: cache_time *= 0.1 elif not is_success(resp.status_int): cache_time = None else: cache_time = None # Next actually set both memcache and the env chache memcache = getattr(app, 'memcache', None) or env.get('swift.cache') if not cache_time: env.pop(env_key, None) if memcache: memcache.delete(cache_key) return if container: info = headers_to_container_info(resp.headers, resp.status_int) else: info = headers_to_account_info(resp.headers, resp.status_int) if memcache: memcache.set(cache_key, info, time=cache_time) env[env_key] = info def _set_object_info_cache(app, env, account, container, obj, resp): """ Cache object info env. Do not cache object informations in memcache. This is an intentional omission as it would lead to cache pressure. This is a per-request cache. Caching is used to avoid unnecessary calls to object servers. This is a private function that is being called by GETorHEAD_base. Any attempt to GET or HEAD from the object server should use the GETorHEAD_base interface which would then set the cache. :param app: the application object :param account: the unquoted account name :param container: the unquoted container name or None :param object: the unquoted object name or None :param resp: the response received or None if info cache should be cleared """ env_key = get_object_env_key(account, container, obj) if not resp: env.pop(env_key, None) return info = headers_to_object_info(resp.headers, resp.status_int) env[env_key] = info def clear_info_cache(app, env, account, container=None): """ Clear the cached info in both memcache and env :param app: the application object :param account: the account name :param container: the containr name or None if setting info for containers """ _set_info_cache(app, env, account, container, None) def _get_info_cache(app, env, account, container=None): """ Get the cached info from env or memcache (if used) in that order Used for both account and container info A private function used by get_info :param app: the application object :param env: the environment used by the current request :returns the cached info or None if not cached """ cache_key, env_key = _get_cache_key(account, container) if env_key in env: return env[env_key] memcache = getattr(app, 'memcache', None) or env.get('swift.cache') if memcache: info = memcache.get(cache_key) if info: for key in info: if isinstance(info[key], unicode): info[key] = info[key].encode("utf-8") env[env_key] = info return info return None def _prepare_pre_auth_info_request(env, path, swift_source): """ Prepares a pre authed request to obtain info using a HEAD. :param env: the environment used by the current request :param path: The unquoted request path :param swift_source: value for swift.source in WSGI environment :returns: the pre authed request """ # Set the env for the pre_authed call without a query string newenv = make_pre_authed_env(env, 'HEAD', path, agent='Swift', query_string='', swift_source=swift_source) # This is a sub request for container metadata- drop the Origin header from # the request so the it is not treated as a CORS request. newenv.pop('HTTP_ORIGIN', None) # Note that Request.blank expects quoted path return Request.blank(quote(path), environ=newenv) def get_info(app, env, account, container=None, ret_not_found=False, swift_source=None): """ Get the info about accounts or containers Note: This call bypasses auth. Success does not imply that the request has authorization to the info. :param app: the application object :param env: the environment used by the current request :param account: The unquoted name of the account :param container: The unquoted name of the container (or None if account) :returns: the cached info or None if cannot be retrieved """ info = _get_info_cache(app, env, account, container) if info: if ret_not_found or is_success(info['status']): return info return None # Not in cache, let's try the account servers path = '/v1/%s' % account if container: # Stop and check if we have an account? if not get_info(app, env, account): return None path += '/' + container req = _prepare_pre_auth_info_request( env, path, (swift_source or 'GET_INFO')) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key] and in memcache. We will # pick the one from environ[env_key] and use it to set the caller env resp = req.get_response(app) cache_key, env_key = _get_cache_key(account, container) try: info = resp.environ[env_key] env[env_key] = info if ret_not_found or is_success(info['status']): return info except (KeyError, AttributeError): pass return None def _get_object_info(app, env, account, container, obj, swift_source=None): """ Get the info about object Note: This call bypasses auth. Success does not imply that the request has authorization to the info. :param app: the application object :param env: the environment used by the current request :param account: The unquoted name of the account :param container: The unquoted name of the container :param obj: The unquoted name of the object :returns: the cached info or None if cannot be retrieved """ env_key = get_object_env_key(account, container, obj) info = env.get(env_key) if info: return info # Not in cached, let's try the object servers path = '/v1/%s/%s/%s' % (account, container, obj) req = _prepare_pre_auth_info_request(env, path, swift_source) # Whenever we do a GET/HEAD, the GETorHEAD_base will set the info in # the environment under environ[env_key]. We will # pick the one from environ[env_key] and use it to set the caller env resp = req.get_response(app) try: info = resp.environ[env_key] env[env_key] = info return info except (KeyError, AttributeError): pass return None def close_swift_conn(src): """ Force close the http connection to the backend. :param src: the response from the backend """ try: # Since the backends set "Connection: close" in their response # headers, the response object (src) is solely responsible for the # socket. The connection object (src.swift_conn) has no references # to the socket, so calling its close() method does nothing, and # therefore we don't do it. # # Also, since calling the response's close() method might not # close the underlying socket but only decrement some # reference-counter, we have a special method here that really, # really kills the underlying socket with a close() syscall. src.nuke_from_orbit() # it's the only way to be sure except Exception: pass class GetOrHeadHandler(object): def __init__(self, app, req, server_type, ring, partition, path, backend_headers): self.app = app self.ring = ring self.server_type = server_type self.partition = partition self.path = path self.backend_headers = backend_headers self.used_nodes = [] self.used_source_etag = '' # stuff from request self.req_method = req.method self.req_path = req.path self.req_query_string = req.query_string self.newest = config_true_value(req.headers.get('x-newest', 'f')) # populated when finding source self.statuses = [] self.reasons = [] self.bodies = [] self.source_headers = [] def fast_forward(self, num_bytes): """ Will skip num_bytes into the current ranges. :params num_bytes: the number of bytes that have already been read on this request. This will change the Range header so that the next req will start where it left off. :raises NotImplementedError: if this is a multirange request :raises ValueError: if invalid range header :raises HTTPRequestedRangeNotSatisfiable: if begin + num_bytes > end of range """ if 'Range' in self.backend_headers: req_range = Range(self.backend_headers['Range']) if len(req_range.ranges) > 1: raise NotImplementedError() begin, end = req_range.ranges.pop() if begin is None: # this is a -50 range req (last 50 bytes of file) end -= num_bytes else: begin += num_bytes if end and begin > end: raise HTTPRequestedRangeNotSatisfiable() req_range.ranges = [(begin, end)] self.backend_headers['Range'] = str(req_range) else: self.backend_headers['Range'] = 'bytes=%d-' % num_bytes def is_good_source(self, src): """ Indicates whether or not the request made to the backend found what it was looking for. :param src: the response from the backend :returns: True if found, False if not """ if self.server_type == 'Object' and src.status == 416: return True return is_success(src.status) or is_redirection(src.status) def _make_app_iter(self, req, node, source): """ Returns an iterator over the contents of the source (via its read func). There is also quite a bit of cleanup to ensure garbage collection works and the underlying socket of the source is closed. :param req: incoming request object :param source: The httplib.Response object this iterator should read from. :param node: The node the source is reading from, for logging purposes. """ try: nchunks = 0 bytes_read_from_source = 0 node_timeout = self.app.node_timeout if self.server_type == 'Object': node_timeout = self.app.recoverable_node_timeout while True: try: with ChunkReadTimeout(node_timeout): chunk = source.read(self.app.object_chunk_size) nchunks += 1 bytes_read_from_source += len(chunk) except ChunkReadTimeout: exc_type, exc_value, exc_traceback = exc_info() if self.newest or self.server_type != 'Object': raise exc_type, exc_value, exc_traceback try: self.fast_forward(bytes_read_from_source) except (NotImplementedError, HTTPException, ValueError): raise exc_type, exc_value, exc_traceback new_source, new_node = self._get_source_and_node() if new_source: self.app.exception_occurred( node, _('Object'), _('Trying to read during GET (retrying)')) # Close-out the connection as best as possible. if getattr(source, 'swift_conn', None): close_swift_conn(source) source = new_source node = new_node bytes_read_from_source = 0 continue else: raise exc_type, exc_value, exc_traceback if not chunk: break with ChunkWriteTimeout(self.app.client_timeout): yield chunk # This is for fairness; if the network is outpacing the CPU, # we'll always be able to read and write data without # encountering an EWOULDBLOCK, and so eventlet will not switch # greenthreads on its own. We do it manually so that clients # don't starve. # # The number 5 here was chosen by making stuff up. It's not # every single chunk, but it's not too big either, so it seemed # like it would probably be an okay choice. # # Note that we may trampoline to other greenthreads more often # than once every 5 chunks, depending on how blocking our # network IO is; the explicit sleep here simply provides a # lower bound on the rate of trampolining. if nchunks % 5 == 0: sleep() except ChunkReadTimeout: self.app.exception_occurred(node, _('Object'), _('Trying to read during GET')) raise except ChunkWriteTimeout: self.app.logger.warn( _('Client did not read from proxy within %ss') % self.app.client_timeout) self.app.logger.increment('client_timeouts') except GeneratorExit: if not req.environ.get('swift.non_client_disconnect'): self.app.logger.warn(_('Client disconnected on read')) except Exception: self.app.logger.exception(_('Trying to send to client')) raise finally: # Close-out the connection as best as possible. if getattr(source, 'swift_conn', None): close_swift_conn(source) def _get_source_and_node(self): self.statuses = [] self.reasons = [] self.bodies = [] self.source_headers = [] sources = [] node_timeout = self.app.node_timeout if self.server_type == 'Object' and not self.newest: node_timeout = self.app.recoverable_node_timeout for node in self.app.iter_nodes(self.ring, self.partition): if node in self.used_nodes: continue start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect( node['ip'], node['port'], node['device'], self.partition, self.req_method, self.path, headers=self.backend_headers, query_string=self.req_query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.app.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % {'method': self.req_method, 'path': self.req_path}) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): self.statuses.append(HTTP_NOT_FOUND) self.reasons.append('') self.bodies.append('') self.source_headers.append('') close_swift_conn(possible_source) else: if self.used_source_etag: src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) if src_headers.get('etag', '').strip('"') != \ self.used_source_etag: self.statuses.append(HTTP_NOT_FOUND) self.reasons.append('') self.bodies.append('') self.source_headers.append('') continue self.statuses.append(possible_source.status) self.reasons.append(possible_source.reason) self.bodies.append('') self.source_headers.append('') sources.append((possible_source, node)) if not self.newest: # one good source is enough break else: self.statuses.append(possible_source.status) self.reasons.append(possible_source.reason) self.bodies.append(possible_source.read()) self.source_headers.append(possible_source.getheaders()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(possible_source.status): self.app.error_occurred( node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % {'status': possible_source.status, 'body': self.bodies[-1][:1024], 'type': self.server_type}) if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() for src, _junk in sources: close_swift_conn(src) self.used_nodes.append(node) src_headers = dict( (k.lower(), v) for k, v in possible_source.getheaders()) self.used_source_etag = src_headers.get('etag', '').strip('"') return source, node return None, None def get_working_response(self, req): source, node = self._get_source_and_node() res = None if source: res = Response(request=req) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(req, node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res class Controller(object): """Base WSGI controller class for the proxy""" server_type = 'Base' # Ensure these are all lowercase pass_through_headers = [] def __init__(self, app): """ Creates a controller attached to an application instance :param app: the application instance """ self.account_name = None self.app = app self.trans_id = '-' self._allowed_methods = None @property def allowed_methods(self): if self._allowed_methods is None: self._allowed_methods = set() all_methods = inspect.getmembers(self, predicate=inspect.ismethod) for name, m in all_methods: if getattr(m, 'publicly_accessible', False): self._allowed_methods.add(name) return self._allowed_methods def _x_remove_headers(self): """ Returns a list of headers that must not be sent to the backend :returns: a list of header """ return [] def transfer_headers(self, src_headers, dst_headers): """ Transfer legal headers from an original client request to dictionary that will be used as headers by the backend request :param src_headers: A dictionary of the original client request headers :param dst_headers: A dictionary of the backend request headers """ st = self.server_type.lower() x_remove = 'x-remove-%s-meta-' % st dst_headers.update((k.lower().replace('-remove', '', 1), '') for k in src_headers if k.lower().startswith(x_remove) or k.lower() in self._x_remove_headers()) dst_headers.update((k.lower(), v) for k, v in src_headers.iteritems() if k.lower() in self.pass_through_headers or is_sys_or_user_meta(st, k)) def generate_request_headers(self, orig_req=None, additional=None, transfer=False): """ Create a list of headers to be used in backend requets :param orig_req: the original request sent by the client to the proxy :param additional: additional headers to send to the backend :param transfer: If True, transfer headers from original client request :returns: a dictionary of headers """ # Use the additional headers first so they don't overwrite the headers # we require. headers = HeaderKeyDict(additional) if additional else HeaderKeyDict() if transfer: self.transfer_headers(orig_req.headers, headers) headers.setdefault('x-timestamp', normalize_timestamp(time.time())) if orig_req: referer = orig_req.as_referer() else: referer = '' headers['x-trans-id'] = self.trans_id headers['connection'] = 'close' headers['user-agent'] = 'proxy-server %s' % os.getpid() headers['referer'] = referer return headers def account_info(self, account, req=None): """ Get account information, and also verify that the account exists. :param account: name of the account to get the info for :param req: caller's HTTP request context object (optional) :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) if req: env = getattr(req, 'environ', {}) else: env = {} info = get_info(self.app, env, account) if not info: return None, None, None if info.get('container_count') is None: container_count = 0 else: container_count = int(info['container_count']) return partition, nodes, container_count def container_info(self, account, container, req=None): """ Get container information and thusly verify container existence. This will also verify account existence. :param account: account name for the container :param container: container name to look up :param req: caller's HTTP request context object (optional) :returns: dict containing at least container partition ('partition'), container nodes ('containers'), container read acl ('read_acl'), container write acl ('write_acl'), and container sync key ('sync_key'). Values are set to None if the container does not exist. """ part, nodes = self.app.container_ring.get_nodes(account, container) if req: env = getattr(req, 'environ', {}) else: env = {} info = get_info(self.app, env, account, container) if not info: info = headers_to_container_info({}, 0) info['partition'] = None info['nodes'] = None else: info['partition'] = part info['nodes'] = nodes return info def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals): """ Iterates over the given node iterator, sending an HTTP request to one node at a time. The first non-informational, non-server-error response is returned. If no non-informational, non-server-error response is received from any of the nodes, returns None. :param nodes: an iterator of the backend server and handoff servers :param part: the partition number :param method: the method to send to the backend :param path: the path to send to the backend (full path ends up being /<$device>/<$part>/<$path>) :param headers: a list of dicts, where each dict represents one backend request that should be made. :param query: query string to send to the backend. :param logger_thread_locals: The thread local values to be set on the self.app.logger to retain transaction logging information. :returns: a swob.Response object, or None if no responses were received """ self.app.logger.thread_locals = logger_thread_locals for node in nodes: try: start_node_timing = time.time() with ConnectionTimeout(self.app.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, method, path, headers=headers, query_string=query) conn.node = node self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): resp = conn.getresponse() if not is_informational(resp.status) and \ not is_server_error(resp.status): return resp.status, resp.reason, resp.getheaders(), \ resp.read() elif resp.status == HTTP_INSUFFICIENT_STORAGE: self.app.error_limit(node, _('ERROR Insufficient Storage')) except (Exception, Timeout): self.app.exception_occurred( node, self.server_type, _('Trying to %(method)s %(path)s') % {'method': method, 'path': path}) def make_requests(self, req, ring, part, method, path, headers, query_string=''): """ Sends an HTTP request to multiple nodes and aggregates the results. It attempts the primary nodes concurrently, then iterates over the handoff nodes as needed. :param req: a request sent by the client :param ring: the ring used for finding backend servers :param part: the partition number :param method: the method to send to the backend :param path: the path to send to the backend (full path ends up being /<$device>/<$part>/<$path>) :param headers: a list of dicts, where each dict represents one backend request that should be made. :param query_string: optional query string to send to the backend :returns: a swob.Response object """ start_nodes = ring.get_part_nodes(part) nodes = GreenthreadSafeIterator(self.app.iter_nodes(ring, part)) pile = GreenAsyncPile(len(start_nodes)) for head in headers: pile.spawn(self._make_request, nodes, part, method, path, head, query_string, self.app.logger.thread_locals) response = [] statuses = [] for resp in pile: if not resp: continue response.append(resp) statuses.append(resp[0]) if self.have_quorum(statuses, len(start_nodes)): break # give any pending requests *some* chance to finish pile.waitall(self.app.post_quorum_timeout) while len(response) < len(start_nodes): response.append((HTTP_SERVICE_UNAVAILABLE, '', '', '')) statuses, reasons, resp_headers, bodies = zip(*response) return self.best_response(req, statuses, reasons, bodies, '%s %s' % (self.server_type, req.method), headers=resp_headers) def have_quorum(self, statuses, node_count): """ Given a list of statuses from several requests, determine if a quorum response can already be decided. :param statuses: list of statuses returned :param node_count: number of nodes being queried (basically ring count) :returns: True or False, depending on if quorum is established """ quorum = quorum_size(node_count) if len(statuses) >= quorum: for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): if sum(1 for s in statuses if hundred <= s < hundred + 100) >= quorum: return True return False def best_response(self, req, statuses, reasons, bodies, server_type, etag=None, headers=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :param headers: headers of each response :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) >= quorum_size(len(statuses)): status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] if headers: update_headers(resp, headers[status_index]) if etag: resp.headers['etag'] = etag.strip('"') return resp self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'), {'type': server_type, 'statuses': statuses}) resp.status = '503 Internal Server Error' return resp @public def GET(self, req): """ Handler for HTTP GET requests. :param req: The client request :returns: the response to the client """ return self.GETorHEAD(req) @public def HEAD(self, req): """ Handler for HTTP HEAD requests. :param req: The client request :returns: the response to the client """ return self.GETorHEAD(req) def autocreate_account(self, env, account): """ Autocreate an account :param env: the environment of the request leading to this autocreate :param account: the unquoted account name """ partition, nodes = self.app.account_ring.get_nodes(account) path = '/%s' % account headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} resp = self.make_requests(Request.blank('/v1' + path), self.app.account_ring, partition, 'PUT', path, [headers] * len(nodes)) if is_success(resp.status_int): self.app.logger.info('autocreate account %r' % path) clear_info_cache(self.app, env, account) else: self.app.logger.warning('Could not autocreate account %r' % path) def GETorHEAD_base(self, req, server_type, ring, partition, path): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type used in logging :param ring: the ring to obtain nodes from :param partition: partition :param path: path for the request :returns: swob.Response object """ backend_headers = self.generate_request_headers( req, additional=req.headers) handler = GetOrHeadHandler(self.app, req, self.server_type, ring, partition, path, backend_headers) res = handler.get_working_response(req) if not res: res = self.best_response( req, handler.statuses, handler.reasons, handler.bodies, '%s %s' % (server_type, req.method), headers=handler.source_headers) try: (vrs, account, container) = req.split_path(2, 3) _set_info_cache(self.app, req.environ, account, container, res) except ValueError: pass try: (vrs, account, container, obj) = req.split_path(4, 4, True) _set_object_info_cache(self.app, req.environ, account, container, obj, res) except ValueError: pass return res def is_origin_allowed(self, cors_info, origin): """ Is the given Origin allowed to make requests to this resource :param cors_info: the resource's CORS related metadata headers :param origin: the origin making the request :return: True or False """ allowed_origins = set() if cors_info.get('allow_origin'): allowed_origins.update( [a.strip() for a in cors_info['allow_origin'].split(' ') if a.strip()]) if self.app.cors_allow_origin: allowed_origins.update(self.app.cors_allow_origin) return origin in allowed_origins or '*' in allowed_origins @public def OPTIONS(self, req): """ Base handler for OPTIONS requests :param req: swob.Request object :returns: swob.Response object """ # Prepare the default response headers = {'Allow': ', '.join(self.allowed_methods)} resp = Response(status=200, request=req, headers=headers) # If this isn't a CORS pre-flight request then return now req_origin_value = req.headers.get('Origin', None) if not req_origin_value: return resp # This is a CORS preflight request so check it's allowed try: container_info = \ self.container_info(self.account_name, self.container_name, req) except AttributeError: # This should only happen for requests to the Account. A future # change could allow CORS requests to the Account level as well. return resp cors = container_info.get('cors', {}) # If the CORS origin isn't allowed return a 401 if not self.is_origin_allowed(cors, req_origin_value) or ( req.headers.get('Access-Control-Request-Method') not in self.allowed_methods): resp.status = HTTP_UNAUTHORIZED return resp # Allow all headers requested in the request. The CORS # specification does leave the door open for this, as mentioned in # http://www.w3.org/TR/cors/#resource-preflight-requests # Note: Since the list of headers can be unbounded # simply returning headers can be enough. allow_headers = set() if req.headers.get('Access-Control-Request-Headers'): allow_headers.update( list_from_csv(req.headers['Access-Control-Request-Headers'])) # Populate the response with the CORS preflight headers if cors.get('allow_origin', '').strip() == '*': headers['access-control-allow-origin'] = '*' else: headers['access-control-allow-origin'] = req_origin_value if cors.get('max_age') is not None: headers['access-control-max-age'] = cors.get('max_age') headers['access-control-allow-methods'] = \ ', '.join(self.allowed_methods) if allow_headers: headers['access-control-allow-headers'] = ', '.join(allow_headers) resp.headers = headers return resp swift-1.13.1/swift/proxy/controllers/info.py0000664000175400017540000000727012323703611022263 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from time import time from swift.common.utils import public, get_hmac, get_swift_info, json, \ streq_const_time from swift.proxy.controllers.base import Controller, delay_denial from swift.common.swob import HTTPOk, HTTPForbidden, HTTPUnauthorized class InfoController(Controller): """WSGI controller for info requests""" server_type = 'Info' def __init__(self, app, version, expose_info, disallowed_sections, admin_key): Controller.__init__(self, app) self.expose_info = expose_info self.disallowed_sections = disallowed_sections self.admin_key = admin_key self.allowed_hmac_methods = { 'HEAD': ['HEAD', 'GET'], 'GET': ['GET']} @public @delay_denial def GET(self, req): return self.GETorHEAD(req) @public @delay_denial def HEAD(self, req): return self.GETorHEAD(req) @public @delay_denial def OPTIONS(self, req): return HTTPOk(request=req, headers={'Allow': 'HEAD, GET, OPTIONS'}) def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" """ Handles requests to /info Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ if not self.expose_info: return HTTPForbidden(request=req) admin_request = False sig = req.params.get('swiftinfo_sig', '') expires = req.params.get('swiftinfo_expires', '') if sig != '' or expires != '': admin_request = True if not self.admin_key: return HTTPForbidden(request=req) try: expires = int(expires) except ValueError: return HTTPUnauthorized(request=req) if expires < time(): return HTTPUnauthorized(request=req) valid_sigs = [] for method in self.allowed_hmac_methods[req.method]: valid_sigs.append(get_hmac(method, '/info', expires, self.admin_key)) # While it's true that any() will short-circuit, this doesn't # affect the timing-attack resistance since the only way this will # short-circuit is when a valid signature is passed in. is_valid_hmac = any(streq_const_time(valid_sig, sig) for valid_sig in valid_sigs) if not is_valid_hmac: return HTTPUnauthorized(request=req) headers = {} if 'Origin' in req.headers: headers['Access-Control-Allow-Origin'] = req.headers['Origin'] headers['Access-Control-Expose-Headers'] = ', '.join( ['x-trans-id']) info = json.dumps(get_swift_info( admin=admin_request, disallowed_sections=self.disallowed_sections)) return HTTPOk(request=req, headers=headers, body=info, content_type='application/json; charset=UTF-8') swift-1.13.1/swift/proxy/controllers/container.py0000664000175400017540000002041412323703614023310 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift import gettext_ as _ from urllib import unquote import time from swift.common.utils import public, csv_append, normalize_timestamp from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH from swift.common.http import HTTP_ACCEPTED from swift.proxy.controllers.base import Controller, delay_denial, \ cors_validation, clear_info_cache from swift.common.swob import HTTPBadRequest, HTTPForbidden, \ HTTPNotFound class ContainerController(Controller): """WSGI controller for container requests""" server_type = 'Container' # Ensure these are all lowercase pass_through_headers = ['x-container-read', 'x-container-write', 'x-container-sync-key', 'x-container-sync-to', 'x-versions-location'] def __init__(self, app, account_name, container_name, **kwargs): Controller.__init__(self, app) self.account_name = unquote(account_name) self.container_name = unquote(container_name) def _x_remove_headers(self): st = self.server_type.lower() return ['x-remove-%s-read' % st, 'x-remove-%s-write' % st, 'x-remove-versions-location'] def clean_acls(self, req): if 'swift.clean_acl' in req.environ: for header in ('x-container-read', 'x-container-write'): if header in req.headers: try: req.headers[header] = \ req.environ['swift.clean_acl'](header, req.headers[header]) except ValueError as err: return HTTPBadRequest(request=req, body=str(err)) return None def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if not self.account_info(self.account_name, req)[1]: return HTTPNotFound(request=req) part = self.app.container_ring.get_part( self.account_name, self.container_name) resp = self.GETorHEAD_base( req, _('Container'), self.app.container_ring, part, req.swift_entity_path) if 'swift.authorize' in req.environ: req.acl = resp.headers.get('x-container-read') aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not req.environ.get('swift_owner', False): for key in self.app.swift_owner_headers: if key in resp.headers: del resp.headers[key] return resp @public @delay_denial @cors_validation def GET(self, req): """Handler for HTTP GET requests.""" return self.GETorHEAD(req) @public @delay_denial @cors_validation def HEAD(self, req): """Handler for HTTP HEAD requests.""" return self.GETorHEAD(req) @public @cors_validation def PUT(self, req): """HTTP PUT request handler.""" error_response = \ self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if not req.environ.get('swift_owner'): for key in self.app.swift_owner_headers: req.headers.pop(key, None) if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH: resp = HTTPBadRequest(request=req) resp.body = 'Container name length of %d longer than %d' % \ (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) return resp account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts and self.app.account_autocreate: self.autocreate_account(req.environ, self.account_name) account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) if self.app.max_containers_per_account > 0 and \ container_count >= self.app.max_containers_per_account and \ self.account_name not in self.app.max_containers_whitelist: resp = HTTPForbidden(request=req) resp.body = 'Reached container limit of %s' % \ self.app.max_containers_per_account return resp container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self._backend_requests(req, len(containers), account_partition, accounts) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'PUT', req.swift_entity_path, headers) return resp @public @cors_validation def POST(self, req): """HTTP POST request handler.""" error_response = \ self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if not req.environ.get('swift_owner'): for key in self.app.swift_owner_headers: req.headers.pop(key, None) account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self.generate_request_headers(req, transfer=True) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'POST', req.swift_entity_path, [headers] * len(containers)) return resp @public @cors_validation def DELETE(self, req): """HTTP DELETE request handler.""" account_partition, accounts, container_count = \ self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = self._backend_requests(req, len(containers), account_partition, accounts) clear_info_cache(self.app, req.environ, self.account_name, self.container_name) resp = self.make_requests( req, self.app.container_ring, container_partition, 'DELETE', req.swift_entity_path, headers) # Indicates no server had the container if resp.status_int == HTTP_ACCEPTED: return HTTPNotFound(request=req) return resp def _backend_requests(self, req, n_outgoing, account_partition, accounts): additional = {'X-Timestamp': normalize_timestamp(time.time())} headers = [self.generate_request_headers(req, transfer=True, additional=additional) for _junk in range(n_outgoing)] for i, account in enumerate(accounts): i = i % len(headers) headers[i]['X-Account-Partition'] = account_partition headers[i]['X-Account-Host'] = csv_append( headers[i].get('X-Account-Host'), '%(ip)s:%(port)s' % account) headers[i]['X-Account-Device'] = csv_append( headers[i].get('X-Account-Device'), account['device']) return headers swift-1.13.1/swift/proxy/controllers/__init__.py0000664000175400017540000000171212323703611023062 0ustar jenkinsjenkins00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.proxy.controllers.base import Controller from swift.proxy.controllers.info import InfoController from swift.proxy.controllers.obj import ObjectController from swift.proxy.controllers.account import AccountController from swift.proxy.controllers.container import ContainerController __all__ = [ 'AccountController', 'ContainerController', 'Controller', 'InfoController', 'ObjectController', ] swift-1.13.1/swift/proxy/__init__.py0000664000175400017540000000000012323703611020501 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/obj/0000775000175400017540000000000012323703665016004 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/obj/mem_diskfile.py0000664000175400017540000003240412323703611021000 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ In-Memory Disk File Interface for Swift Object Server""" import cStringIO import time import hashlib from contextlib import contextmanager from eventlet import Timeout from swift.common.utils import normalize_timestamp from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileCollision, DiskFileDeleted, DiskFileNotOpen from swift.common.swob import multi_range_iterator class InMemoryFileSystem(object): """ A very simplistic in-memory file system scheme. There is one dictionary mapping a given object name to a tuple. The first entry in the tuble is the cStringIO buffer representing the file contents, the second entry is the metadata dictionary. """ def __init__(self): self._filesystem = {} def get_object(self, name): val = self._filesystem.get(name) if val is None: data, metadata = None, None else: data, metadata = val return data, metadata def put_object(self, name, data, metadata): self._filesystem[name] = (data, metadata) def del_object(self, name): del self._filesystem[name] def get_diskfile(self, account, container, obj, **kwargs): return DiskFile(self, account, container, obj) class DiskFileWriter(object): """ .. note:: Sample alternative pluggable on-disk backend implementation. Encapsulation of the write context for servicing PUT REST API requests. Serves as the context manager object for DiskFile's create() method. :param fs: internal file system object to use :param name: standard object name :param fp: `StringIO` in-memory representation object """ def __init__(self, fs, name, fp): self._filesystem = fs self._name = name self._fp = fp self._upload_size = 0 def write(self, chunk): """ Write a chunk of data into the `StringIO` object. :param chunk: the chunk of data to write as a string object """ self._fp.write(chunk) self._upload_size += len(chunk) return self._upload_size def put(self, metadata): """ Make the final association in the in-memory file system for this name with the `StringIO` object. :param metadata: dictionary of metadata to be written :param extension: extension to be used when making the file """ metadata['name'] = self._name self._filesystem.put_object(self._name, self._fp, metadata) class DiskFileReader(object): """ .. note:: Sample alternative pluggable on-disk backend implementation. Encapsulation of the read context for servicing GET REST API requests. Serves as the context manager object for DiskFile's reader() method. :param name: object name :param fp: open file object pointer reference :param obj_size: on-disk size of object in bytes :param etag: MD5 hash of object from metadata """ def __init__(self, name, fp, obj_size, etag): self._name = name self._fp = fp self._obj_size = obj_size self._etag = etag # self._iter_etag = None self._bytes_read = 0 self._started_at_0 = False self._read_to_eof = False self._suppress_file_closing = False # self.was_quarantined = '' def __iter__(self): try: self._bytes_read = 0 self._started_at_0 = False self._read_to_eof = False if self._fp.tell() == 0: self._started_at_0 = True self._iter_etag = hashlib.md5() while True: chunk = self._fp.read() if chunk: if self._iter_etag: self._iter_etag.update(chunk) self._bytes_read += len(chunk) yield chunk else: self._read_to_eof = True break finally: if not self._suppress_file_closing: self.close() def app_iter_range(self, start, stop): if start or start == 0: self._fp.seek(start) if stop is not None: length = stop - start else: length = None try: for chunk in self: if length is not None: length -= len(chunk) if length < 0: # Chop off the extra: yield chunk[:length] break yield chunk finally: if not self._suppress_file_closing: self.close() def app_iter_ranges(self, ranges, content_type, boundary, size): if not ranges: yield '' else: try: self._suppress_file_closing = True for chunk in multi_range_iterator( ranges, content_type, boundary, size, self.app_iter_range): yield chunk finally: self._suppress_file_closing = False try: self.close() except DiskFileQuarantined: pass def _quarantine(self, msg): self.was_quarantined = msg def _handle_close_quarantine(self): if self._bytes_read != self._obj_size: self._quarantine( "Bytes read: %s, does not match metadata: %s" % ( self.bytes_read, self._obj_size)) elif self._iter_etag and \ self._etag != self._iter_etag.hexdigest(): self._quarantine( "ETag %s and file's md5 %s do not match" % ( self._etag, self._iter_etag.hexdigest())) def close(self): """ Close the file. Will handle quarantining file if necessary. """ if self._fp: try: if self._started_at_0 and self._read_to_eof: self._handle_close_quarantine() except (Exception, Timeout): pass finally: self._fp = None class DiskFile(object): """ .. note:: Sample alternative pluggable on-disk backend implementation. This example duck-types the reference implementation DiskFile class. Manage object files in-memory. :param mgr: DiskFileManager :param device_path: path to the target device or drive :param threadpool: thread pool to use for blocking operations :param partition: partition on the device in which the object lives :param account: account name for the object :param container: container name for the object :param obj: object name for the object :param keep_cache: caller's preference for keeping data read in the cache """ def __init__(self, fs, account, container, obj): self._name = '/' + '/'.join((account, container, obj)) self._metadata = None self._fp = None self._filesystem = fs def open(self): """ Open the file and read the metadata. This method must populate the _metadata attribute. :raises DiskFileCollision: on name mis-match with metadata :raises DiskFileDeleted: if it does not exist, or a tombstone is present :raises DiskFileQuarantined: if while reading metadata of the file some data did pass cross checks """ fp, self._metadata = self._filesystem.get_object(self._name) if fp is None: raise DiskFileDeleted() self._fp = self._verify_data_file(fp) self._metadata = self._metadata or {} return self def __enter__(self): if self._metadata is None: raise DiskFileNotOpen() return self def __exit__(self, t, v, tb): if self._fp is not None: self._fp = None def _verify_data_file(self, fp): """ Verify the metadata's name value matches what we think the object is named. :raises DiskFileCollision: if the metadata stored name does not match the referenced name of the file :raises DiskFileNotExist: if the object has expired :raises DiskFileQuarantined: if data inconsistencies were detected between the metadata and the file-system metadata """ try: mname = self._metadata['name'] except KeyError: raise self._quarantine(self._name, "missing name metadata") else: if mname != self._name: raise DiskFileCollision('Client path does not match path ' 'stored in object metadata') try: x_delete_at = int(self._metadata['X-Delete-At']) except KeyError: pass except ValueError: # Quarantine, the x-delete-at key is present but not an # integer. raise self._quarantine( self._name, "bad metadata x-delete-at value %s" % ( self._metadata['X-Delete-At'])) else: if x_delete_at <= time.time(): raise DiskFileNotExist('Expired') try: metadata_size = int(self._metadata['Content-Length']) except KeyError: raise self._quarantine( self._name, "missing content-length in metadata") except ValueError: # Quarantine, the content-length key is present but not an # integer. raise self._quarantine( self._name, "bad metadata content-length value %s" % ( self._metadata['Content-Length'])) try: fp.seek(0, 2) obj_size = fp.tell() fp.seek(0, 0) except OSError as err: # Quarantine, we can't successfully stat the file. raise self._quarantine(self._name, "not stat-able: %s" % err) if obj_size != metadata_size: raise self._quarantine( self._name, "metadata content-length %s does" " not match actual object size %s" % ( metadata_size, obj_size)) return fp def get_metadata(self): """ Provide the metadata for an object as a dictionary. :returns: object's metadata dictionary """ if self._metadata is None: raise DiskFileNotOpen() return self._metadata def read_metadata(self): """ Return the metadata for an object. :returns: metadata dictionary for an object """ with self.open(): return self.get_metadata() def reader(self, keep_cache=False): """ Return a swift.common.swob.Response class compatible "app_iter" object. The responsibility of closing the open file is passed to the DiskFileReader object. :param keep_cache: """ dr = DiskFileReader(self._name, self._fp, int(self._metadata['Content-Length']), self._metadata['ETag']) # At this point the reader object is now responsible for # the file pointer. self._fp = None return dr @contextmanager def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails """ fp = cStringIO.StringIO() try: yield DiskFileWriter(self._filesystem, self._name, fp) finally: del fp def write_metadata(self, metadata): """ Write a block of metadata to an object. """ cur_fp = self._filesystem.get(self._name) if cur_fp is not None: self._filesystem[self._name] = (cur_fp, metadata) def delete(self, timestamp): """ Perform a delete for the given object in the given container under the given account. This creates a tombstone file with the given timestamp, and removes any older versions of the object file. Any file that has an older timestamp than timestamp will be deleted. :param timestamp: timestamp to compare with each file """ timestamp = normalize_timestamp(timestamp) fp, md = self._filesystem.get_object(self._name) if md['X-Timestamp'] < timestamp: self._filesystem.del_object(self._name) swift-1.13.1/swift/obj/replicator.py0000664000175400017540000005550212323703611020520 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os from os.path import isdir, isfile, join import random import shutil import time import itertools import cPickle as pickle from swift import gettext_ as _ import eventlet from eventlet import GreenPool, tpool, Timeout, sleep, hubs from eventlet.green import subprocess from eventlet.support.greenlets import GreenletExit from swift.common.ring import Ring from swift.common.utils import whataremyips, unlink_older_than, \ compute_eta, get_logger, dump_recon_cache, ismount, \ rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \ tpool_reraise, config_auto_int_value from swift.common.bufferedhttp import http_connect from swift.common.daemon import Daemon from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE from swift.obj import ssync_sender from swift.obj.diskfile import DiskFileManager, get_hashes hubs.use_hub(get_hub()) class ObjectReplicator(Daemon): """ Replicate objects. Encapsulates most logic and data needed by the object replication process. Each call to .replicate() performs one replication pass. It's up to the caller to do this in a loop. """ def __init__(self, conf): """ :param conf: configuration object obtained from ConfigParser :param logger: logging object """ self.conf = conf self.logger = get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.port = int(conf.get('bind_port', 6000)) self.concurrency = int(conf.get('concurrency', 1)) self.stats_interval = int(conf.get('stats_interval', '300')) self.object_ring = Ring(self.swift_dir, ring_name='object') self.ring_check_interval = int(conf.get('ring_check_interval', 15)) self.next_check = time.time() + self.ring_check_interval self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.partition_times = [] self.run_pause = int(conf.get('run_pause', 30)) self.rsync_timeout = int(conf.get('rsync_timeout', 900)) self.rsync_io_timeout = conf.get('rsync_io_timeout', '30') self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.http_timeout = int(conf.get('http_timeout', 60)) self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.node_timeout = float(conf.get('node_timeout', 10)) self.sync_method = getattr(self, conf.get('sync_method') or 'rsync') self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.headers = { 'Content-Length': '0', 'user-agent': 'obj-replicator %s' % os.getpid()} self.rsync_error_log_line_length = \ int(conf.get('rsync_error_log_line_length', 0)) self.handoffs_first = config_true_value(conf.get('handoffs_first', False)) self.handoff_delete = config_auto_int_value( conf.get('handoff_delete', 'auto'), 0) self._diskfile_mgr = DiskFileManager(conf, self.logger) def sync(self, node, job, suffixes): # Just exists for doc anchor point """ Synchronize local suffix directories from a partition with a remote node. :param node: the "dev" entry for the remote node to sync with :param job: information about the partition being synced :param suffixes: a list of suffixes which need to be pushed :returns: boolean indicating success or failure """ return self.sync_method(node, job, suffixes) def _rsync(self, args): """ Execute the rsync binary to replicate a partition. :returns: return code of rsync process. 0 is successful """ start_time = time.time() ret_val = None try: with Timeout(self.rsync_timeout): proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() except Timeout: self.logger.error(_("Killing long-running rsync: %s"), str(args)) proc.kill() return 1 # failure response code total_time = time.time() - start_time for result in results.split('\n'): if result == '': continue if result.startswith('cd+'): continue if not ret_val: self.logger.info(result) else: self.logger.error(result) if ret_val: error_line = _('Bad rsync return code: %(ret)d <- %(args)s') % \ {'args': str(args), 'ret': ret_val} if self.rsync_error_log_line_length: error_line = error_line[:self.rsync_error_log_line_length] self.logger.error(error_line) elif results: self.logger.info( _("Successful rsync of %(src)s at %(dst)s (%(time).03f)"), {'src': args[-2], 'dst': args[-1], 'time': total_time}) else: self.logger.debug( _("Successful rsync of %(src)s at %(dst)s (%(time).03f)"), {'src': args[-2], 'dst': args[-1], 'time': total_time}) return ret_val def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append(join(rsync_module, node['device'], 'objects', job['partition'])) return self._rsync(args) == 0 def ssync(self, node, job, suffixes): return ssync_sender.Sender(self, node, job, suffixes)() def check_ring(self): """ Check to see if the ring has been updated :returns: boolean indicating whether or not the ring has changed """ if time.time() > self.next_check: self.next_check = time.time() + self.ring_check_interval if self.object_ring.has_changed(): return False return True def update_deleted(self, job): """ High-level method that replicates a single partition that doesn't belong on this node. :param job: a dict containing info about the partition to be replicated """ def tpool_get_suffixes(path): return [suff for suff in os.listdir(path) if len(suff) == 3 and isdir(join(path, suff))] self.replication_count += 1 self.logger.increment('partition.delete.count.%s' % (job['device'],)) begin = time.time() try: responses = [] suffixes = tpool.execute(tpool_get_suffixes, job['path']) if suffixes: for node in job['nodes']: success = self.sync(node, job, suffixes) if success: with Timeout(self.http_timeout): conn = http_connect( node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() responses.append(success) if self.handoff_delete: # delete handoff if we have had handoff_delete successes delete_handoff = len([resp for resp in responses if resp]) >= \ self.handoff_delete else: # delete handoff if all syncs were successful delete_handoff = len(responses) == len(job['nodes']) and \ all(responses) if not suffixes or delete_handoff: self.logger.info(_("Removing partition: %s"), job['path']) tpool.execute(shutil.rmtree, job['path'], ignore_errors=True) except (Exception, Timeout): self.logger.exception(_("Error syncing handoff partition")) finally: self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.delete.timing', begin) def update(self, job): """ High-level method that replicates a single partition. :param job: a dict containing info about the partition to be replicated """ self.replication_count += 1 self.logger.increment('partition.update.count.%s' % (job['device'],)) begin = time.time() try: hashed, local_hash = tpool_reraise( get_hashes, job['path'], do_listdir=(self.replication_count % 10) == 0, reclaim_age=self.reclaim_age) self.suffix_hash += hashed self.logger.update_stats('suffix.hashes', hashed) attempts_left = len(job['nodes']) nodes = itertools.chain( job['nodes'], self.object_ring.get_more_nodes(int(job['partition']))) while attempts_left > 0: # If this throws StopIterator it will be caught way below node = next(nodes) attempts_left -= 1 try: with Timeout(self.http_timeout): resp = http_connect( node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '', headers=self.headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error(_('%(ip)s/%(device)s responded' ' as unmounted'), node) attempts_left += 1 continue if resp.status != HTTP_OK: self.logger.error(_("Invalid response %(resp)s " "from %(ip)s"), {'resp': resp.status, 'ip': node['replication_ip']}) continue remote_hash = pickle.loads(resp.read()) del resp suffixes = [suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1)] if not suffixes: continue hashed, recalc_hash = tpool_reraise( get_hashes, job['path'], recalculate=suffixes, reclaim_age=self.reclaim_age) self.logger.update_stats('suffix.hashes', hashed) local_hash = recalc_hash suffixes = [suffix for suffix in local_hash if local_hash[suffix] != remote_hash.get(suffix, -1)] self.sync(node, job, suffixes) with Timeout(self.http_timeout): conn = http_connect( node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=self.headers) conn.getresponse().read() self.suffix_sync += len(suffixes) self.logger.update_stats('suffix.syncs', len(suffixes)) except (Exception, Timeout): self.logger.exception(_("Error syncing with node: %s") % node) self.suffix_count += len(local_hash) except (Exception, Timeout): self.logger.exception(_("Error syncing partition")) finally: self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.update.timing', begin) def stats_line(self): """ Logs various stats for the currently running replication pass. """ if self.replication_count: elapsed = (time.time() - self.start) or 0.000001 rate = self.replication_count / elapsed self.logger.info( _("%(replicated)d/%(total)d (%(percentage).2f%%)" " partitions replicated in %(time).2fs (%(rate).2f/sec, " "%(remaining)s remaining)"), {'replicated': self.replication_count, 'total': self.job_count, 'percentage': self.replication_count * 100.0 / self.job_count, 'time': time.time() - self.start, 'rate': rate, 'remaining': '%d%s' % compute_eta(self.start, self.replication_count, self.job_count)}) if self.suffix_count: self.logger.info( _("%(checked)d suffixes checked - " "%(hashed).2f%% hashed, %(synced).2f%% synced"), {'checked': self.suffix_count, 'hashed': (self.suffix_hash * 100.0) / self.suffix_count, 'synced': (self.suffix_sync * 100.0) / self.suffix_count}) self.partition_times.sort() self.logger.info( _("Partition times: max %(max).4fs, " "min %(min).4fs, med %(med).4fs"), {'max': self.partition_times[-1], 'min': self.partition_times[0], 'med': self.partition_times[ len(self.partition_times) // 2]}) else: self.logger.info( _("Nothing replicated for %s seconds."), (time.time() - self.start)) def kill_coros(self): """Utility function that kills all coroutines currently running.""" for coro in list(self.run_pool.coroutines_running): try: coro.kill(GreenletExit) except GreenletExit: pass def heartbeat(self): """ Loop that runs in the background during replication. It periodically logs progress. """ while True: eventlet.sleep(self.stats_interval) self.stats_line() def detect_lockups(self): """ In testing, the pool.waitall() call very occasionally failed to return. This is an attempt to make sure the replicator finishes its replication pass in some eventuality. """ while True: eventlet.sleep(self.lockup_timeout) if self.replication_count == self.last_replication_count: self.logger.error(_("Lockup detected.. killing live coros.")) self.kill_coros() self.last_replication_count = self.replication_count def collect_jobs(self): """ Returns a sorted list of jobs (dictionaries) that specify the partitions, nodes, etc to be synced. """ jobs = [] ips = whataremyips() for local_dev in [dev for dev in self.object_ring.devs if dev and dev['replication_ip'] in ips and dev['replication_port'] == self.port]: dev_path = join(self.devices_dir, local_dev['device']) obj_path = join(dev_path, 'objects') tmp_path = join(dev_path, 'tmp') if self.mount_check and not ismount(dev_path): self.logger.warn(_('%s is not mounted'), local_dev['device']) continue unlink_older_than(tmp_path, time.time() - self.reclaim_age) if not os.path.exists(obj_path): try: mkdirs(obj_path) except Exception: self.logger.exception('ERROR creating %s' % obj_path) continue for partition in os.listdir(obj_path): try: job_path = join(obj_path, partition) if isfile(job_path): # Clean up any (probably zero-byte) files where a # partition should be. self.logger.warning('Removing partition directory ' 'which was a file: %s', job_path) os.remove(job_path) continue part_nodes = \ self.object_ring.get_part_nodes(int(partition)) nodes = [node for node in part_nodes if node['id'] != local_dev['id']] jobs.append( dict(path=job_path, device=local_dev['device'], nodes=nodes, delete=len(nodes) > len(part_nodes) - 1, partition=partition)) except (ValueError, OSError): continue random.shuffle(jobs) if self.handoffs_first: # Move the handoff parts to the front of the list jobs.sort(key=lambda job: not job['delete']) self.job_count = len(jobs) return jobs def replicate(self, override_devices=None, override_partitions=None): """Run a replication pass""" self.start = time.time() self.suffix_count = 0 self.suffix_sync = 0 self.suffix_hash = 0 self.replication_count = 0 self.last_replication_count = -1 self.partition_times = [] if override_devices is None: override_devices = [] if override_partitions is None: override_partitions = [] stats = eventlet.spawn(self.heartbeat) lockup_detector = eventlet.spawn(self.detect_lockups) eventlet.sleep() # Give spawns a cycle try: self.run_pool = GreenPool(size=self.concurrency) jobs = self.collect_jobs() for job in jobs: if override_devices and job['device'] not in override_devices: continue if override_partitions and \ job['partition'] not in override_partitions: continue dev_path = join(self.devices_dir, job['device']) if self.mount_check and not ismount(dev_path): self.logger.warn(_('%s is not mounted'), job['device']) continue if not self.check_ring(): self.logger.info(_("Ring change detected. Aborting " "current replication pass.")) return if job['delete']: self.run_pool.spawn(self.update_deleted, job) else: self.run_pool.spawn(self.update, job) with Timeout(self.lockup_timeout): self.run_pool.waitall() except (Exception, Timeout): self.logger.exception(_("Exception in top-level replication loop")) self.kill_coros() finally: stats.kill() lockup_detector.kill() self.stats_line() def run_once(self, *args, **kwargs): start = time.time() self.logger.info(_("Running object replicator in script mode.")) override_devices = list_from_csv(kwargs.get('devices')) override_partitions = list_from_csv(kwargs.get('partitions')) self.replicate( override_devices=override_devices, override_partitions=override_partitions) total = (time.time() - start) / 60 self.logger.info( _("Object replication complete (once). (%.02f minutes)"), total) if not (override_partitions or override_devices): dump_recon_cache({'object_replication_time': total, 'object_replication_last': time.time()}, self.rcache, self.logger) def run_forever(self, *args, **kwargs): self.logger.info(_("Starting object replicator in daemon mode.")) # Run the replicator continually while True: start = time.time() self.logger.info(_("Starting object replication pass.")) # Run the replicator self.replicate() total = (time.time() - start) / 60 self.logger.info( _("Object replication complete. (%.02f minutes)"), total) dump_recon_cache({'object_replication_time': total, 'object_replication_last': time.time()}, self.rcache, self.logger) self.logger.debug(_('Replication sleeping for %s seconds.'), self.run_pause) sleep(self.run_pause) swift-1.13.1/swift/obj/ssync_sender.py0000664000175400017540000003104512323703611021047 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import urllib from swift.common import bufferedhttp from swift.common import exceptions from swift.common import http class Sender(object): """ Sends REPLICATION requests to the object server. These requests are eventually handled by :py:mod:`.ssync_receiver` and full documentation about the process is there. """ def __init__(self, daemon, node, job, suffixes): self.daemon = daemon self.node = node self.job = job self.suffixes = suffixes self.connection = None self.response = None self.response_buffer = '' self.response_chunk_left = 0 self.send_list = None self.failures = 0 def __call__(self): if not self.suffixes: return True try: # Double try blocks in case our main error handler fails. try: # The general theme for these functions is that they should # raise exceptions.MessageTimeout for client timeouts and # exceptions.ReplicationException for common issues that will # abort the replication attempt and log a simple error. All # other exceptions will be logged with a full stack trace. self.connect() self.missing_check() self.updates() self.disconnect() return self.failures == 0 except (exceptions.MessageTimeout, exceptions.ReplicationException) as err: self.daemon.logger.error( '%s:%s/%s/%s %s', self.node.get('ip'), self.node.get('port'), self.node.get('device'), self.job.get('partition'), err) except Exception: # We don't want any exceptions to escape our code and possibly # mess up the original replicator code that called us since it # was originally written to shell out to rsync which would do # no such thing. self.daemon.logger.exception( '%s:%s/%s/%s EXCEPTION in replication.Sender', self.node.get('ip'), self.node.get('port'), self.node.get('device'), self.job.get('partition')) except Exception: # We don't want any exceptions to escape our code and possibly # mess up the original replicator code that called us since it # was originally written to shell out to rsync which would do # no such thing. # This particular exception handler does the minimal amount as it # would only get called if the above except Exception handler # failed (bad node or job data). self.daemon.logger.exception('EXCEPTION in replication.Sender') return False def connect(self): """ Establishes a connection and starts a REPLICATION request with the object server. """ with exceptions.MessageTimeout( self.daemon.conn_timeout, 'connect send'): self.connection = bufferedhttp.BufferedHTTPConnection( '%s:%s' % (self.node['ip'], self.node['port'])) self.connection.putrequest('REPLICATION', '/%s/%s' % ( self.node['device'], self.job['partition'])) self.connection.putheader('Transfer-Encoding', 'chunked') self.connection.endheaders() with exceptions.MessageTimeout( self.daemon.node_timeout, 'connect receive'): self.response = self.connection.getresponse() if self.response.status != http.HTTP_OK: raise exceptions.ReplicationException( 'Expected status %s; got %s' % (http.HTTP_OK, self.response.status)) def readline(self): """ Reads a line from the REPLICATION response body. httplib has no readline and will block on read(x) until x is read, so we have to do the work ourselves. A bit of this is taken from Python's httplib itself. """ data = self.response_buffer self.response_buffer = '' while '\n' not in data and len(data) < self.daemon.network_chunk_size: if self.response_chunk_left == -1: # EOF-already indicator break if self.response_chunk_left == 0: line = self.response.fp.readline() i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions try: self.response_chunk_left = int(line.strip(), 16) except ValueError: # close the connection as protocol synchronisation is # probably lost self.response.close() raise exceptions.ReplicationException('Early disconnect') if self.response_chunk_left == 0: self.response_chunk_left = -1 break chunk = self.response.fp.read(min( self.response_chunk_left, self.daemon.network_chunk_size - len(data))) if not chunk: # close the connection as protocol synchronisation is # probably lost self.response.close() raise exceptions.ReplicationException('Early disconnect') self.response_chunk_left -= len(chunk) if self.response_chunk_left == 0: self.response.fp.read(2) # discard the trailing \r\n data += chunk if '\n' in data: data, self.response_buffer = data.split('\n', 1) data += '\n' return data def missing_check(self): """ Handles the sender-side of the MISSING_CHECK step of a REPLICATION request. Full documentation of this can be found at :py:meth:`.Receiver.missing_check`. """ # First, send our list. with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check start'): msg = ':MISSING_CHECK: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for path, object_hash, timestamp in \ self.daemon._diskfile_mgr.yield_hashes( self.job['device'], self.job['partition'], self.suffixes): with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check send line'): msg = '%s %s\r\n' % ( urllib.quote(object_hash), urllib.quote(timestamp)) self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check end'): msg = ':MISSING_CHECK: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, retrieve the list of what they want. while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'missing_check start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) self.send_list = [] while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'missing_check line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: END': break if line: self.send_list.append(line) def updates(self): """ Handles the sender-side of the UPDATES step of a REPLICATION request. Full documentation of this can be found at :py:meth:`.Receiver.updates`. """ # First, send all our subrequests based on the send_list. with exceptions.MessageTimeout( self.daemon.node_timeout, 'updates start'): msg = ':UPDATES: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for object_hash in self.send_list: try: df = self.daemon._diskfile_mgr.get_diskfile_from_hash( self.job['device'], self.job['partition'], object_hash) except exceptions.DiskFileNotExist: continue url_path = urllib.quote( '/%s/%s/%s' % (df.account, df.container, df.obj)) try: df.open() except exceptions.DiskFileDeleted as err: self.send_delete(url_path, err.timestamp) except exceptions.DiskFileError: pass else: self.send_put(url_path, df) with exceptions.MessageTimeout( self.daemon.node_timeout, 'updates end'): msg = ':UPDATES: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, read their response for any issues. while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'updates start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'updates line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: END': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) def send_delete(self, url_path, timestamp): """ Sends a DELETE subrequest with the given information. """ msg = ['DELETE ' + url_path, 'X-Timestamp: ' + timestamp] msg = '\r\n'.join(msg) + '\r\n\r\n' with exceptions.MessageTimeout( self.daemon.node_timeout, 'send_delete'): self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) def send_put(self, url_path, df): """ Sends a PUT subrequest for the url_path using the source df (DiskFile) and content_length. """ msg = ['PUT ' + url_path, 'Content-Length: ' + str(df.content_length)] # Sorted to make it easier to test. for key, value in sorted(df.get_metadata().iteritems()): if key not in ('name', 'Content-Length'): msg.append('%s: %s' % (key, value)) msg = '\r\n'.join(msg) + '\r\n\r\n' with exceptions.MessageTimeout(self.daemon.node_timeout, 'send_put'): self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for chunk in df.reader(): with exceptions.MessageTimeout( self.daemon.node_timeout, 'send_put chunk'): self.connection.send('%x\r\n%s\r\n' % (len(chunk), chunk)) def disconnect(self): """ Closes down the connection to the object server once done with the REPLICATION request. """ try: with exceptions.MessageTimeout( self.daemon.node_timeout, 'disconnect'): self.connection.send('0\r\n\r\n') except (Exception, exceptions.Timeout): pass # We're okay with the above failing. self.connection.close() swift-1.13.1/swift/obj/ssync_receiver.py0000664000175400017540000004034512323703611021376 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import urllib import eventlet import eventlet.wsgi import eventlet.greenio from swift.common import constraints from swift.common import exceptions from swift.common import http from swift.common import swob from swift.common import utils class Receiver(object): """ Handles incoming REPLICATION requests to the object server. These requests come from the object-replicator daemon that uses :py:mod:`.ssync_sender`. The number of concurrent REPLICATION requests is restricted by use of a replication_semaphore and can be configured with the object-server.conf [object-server] replication_concurrency setting. A REPLICATION request is really just an HTTP conduit for sender/receiver replication communication. The overall REPLICATION request should always succeed, but it will contain multiple requests within its request and response bodies. This "hack" is done so that replication concurrency can be managed. The general process inside a REPLICATION request is: 1. Initialize the request: Basic request validation, mount check, acquire semaphore lock, etc.. 2. Missing check: Sender sends the hashes and timestamps of the object information it can send, receiver sends back the hashes it wants (doesn't have or has an older timestamp). 3. Updates: Sender sends the object information requested. 4. Close down: Release semaphore lock, etc. """ def __init__(self, app, request): self.app = app self.request = request self.device = None self.partition = None self.fp = None # We default to dropping the connection in case there is any exception # raised during processing because otherwise the sender could send for # quite some time before realizing it was all in vain. self.disconnect = True def __call__(self): """ Processes a REPLICATION request. Acquires a semaphore lock and then proceeds through the steps of the REPLICATION process. """ # The general theme for functions __call__ calls is that they should # raise exceptions.MessageTimeout for client timeouts (logged locally), # swob.HTTPException classes for exceptions to return to the caller but # not log locally (unmounted, for example), and any other Exceptions # will be logged with a full stack trace. # This is because the client is never just some random user but # is instead also our code and we definitely want to know if our code # is broken or doing something unexpected. try: # Double try blocks in case our main error handlers fail. try: # intialize_request is for preamble items that can be done # outside a replication semaphore lock. for data in self.initialize_request(): yield data # If semaphore is in use, try to acquire it, non-blocking, and # return a 503 if it fails. if self.app.replication_semaphore: if not self.app.replication_semaphore.acquire(False): raise swob.HTTPServiceUnavailable() try: with self.app._diskfile_mgr.replication_lock(self.device): for data in self.missing_check(): yield data for data in self.updates(): yield data # We didn't raise an exception, so end the request # normally. self.disconnect = False finally: if self.app.replication_semaphore: self.app.replication_semaphore.release() except exceptions.ReplicationLockTimeout as err: self.app.logger.debug( '%s/%s/%s REPLICATION LOCK TIMEOUT: %s' % ( self.request.remote_addr, self.device, self.partition, err)) yield ':ERROR: %d %r\n' % (0, str(err)) except exceptions.MessageTimeout as err: self.app.logger.error( '%s/%s/%s TIMEOUT in replication.Receiver: %s' % ( self.request.remote_addr, self.device, self.partition, err)) yield ':ERROR: %d %r\n' % (408, str(err)) except swob.HTTPException as err: body = ''.join(err({}, lambda *args: None)) yield ':ERROR: %d %r\n' % (err.status_int, body) except Exception as err: self.app.logger.exception( '%s/%s/%s EXCEPTION in replication.Receiver' % (self.request.remote_addr, self.device, self.partition)) yield ':ERROR: %d %r\n' % (0, str(err)) except Exception: self.app.logger.exception('EXCEPTION in replication.Receiver') if self.disconnect: # This makes the socket close early so the remote side doesn't have # to send its whole request while the lower Eventlet-level just # reads it and throws it away. Instead, the connection is dropped # and the remote side will get a broken-pipe exception. try: socket = self.request.environ['wsgi.input'].get_socket() eventlet.greenio.shutdown_safe(socket) socket.close() except Exception: pass # We're okay with the above failing. def _ensure_flush(self): """ Sends a blank line sufficient to flush buffers. This is to ensure Eventlet versions that don't support eventlet.minimum_write_chunk_size will send any previous data buffered. If https://bitbucket.org/eventlet/eventlet/pull-request/37 ever gets released in an Eventlet version, we should make this yield only for versions older than that. """ yield ' ' * eventlet.wsgi.MINIMUM_CHUNK_SIZE + '\r\n' def initialize_request(self): """ Basic validation of request and mount check. This function will be called before attempting to acquire a replication semaphore lock, so contains only quick checks. """ # The following is the setting we talk about above in _ensure_flush. self.request.environ['eventlet.minimum_write_chunk_size'] = 0 self.device, self.partition = utils.split_path( urllib.unquote(self.request.path), 2, 2, False) utils.validate_device_partition(self.device, self.partition) if self.app._diskfile_mgr.mount_check and \ not constraints.check_mount( self.app._diskfile_mgr.devices, self.device): raise swob.HTTPInsufficientStorage(drive=self.device) self.fp = self.request.environ['wsgi.input'] for data in self._ensure_flush(): yield data def missing_check(self): """ Handles the receiver-side of the MISSING_CHECK step of a REPLICATION request. Receives a list of hashes and timestamps of object information the sender can provide and responds with a list of hashes desired, either because they're missing or have an older timestamp locally. The process is generally: 1. Sender sends `:MISSING_CHECK: START` and begins sending `hash timestamp` lines. 2. Receiver gets `:MISSING_CHECK: START` and begins reading the `hash timestamp` lines, collecting the hashes of those it desires. 3. Sender sends `:MISSING_CHECK: END`. 4. Receiver gets `:MISSING_CHECK: END`, responds with `:MISSING_CHECK: START`, followed by the list of hashes it collected as being wanted (one per line), `:MISSING_CHECK: END`, and flushes any buffers. 5. Sender gets `:MISSING_CHECK: START` and reads the list of hashes desired by the receiver until reading `:MISSING_CHECK: END`. The collection and then response is so the sender doesn't have to read while it writes to ensure network buffers don't fill up and block everything. """ with exceptions.MessageTimeout( self.app.client_timeout, 'missing_check start'): line = self.fp.readline(self.app.network_chunk_size) if line.strip() != ':MISSING_CHECK: START': raise Exception( 'Looking for :MISSING_CHECK: START got %r' % line[:1024]) object_hashes = [] while True: with exceptions.MessageTimeout( self.app.client_timeout, 'missing_check line'): line = self.fp.readline(self.app.network_chunk_size) if not line or line.strip() == ':MISSING_CHECK: END': break object_hash, timestamp = [urllib.unquote(v) for v in line.split()] want = False try: df = self.app._diskfile_mgr.get_diskfile_from_hash( self.device, self.partition, object_hash) except exceptions.DiskFileNotExist: want = True else: try: df.open() except exceptions.DiskFileDeleted as err: want = err.timestamp < timestamp except exceptions.DiskFileError, err: want = True else: want = df.timestamp < timestamp if want: object_hashes.append(object_hash) yield ':MISSING_CHECK: START\r\n' yield '\r\n'.join(object_hashes) yield '\r\n' yield ':MISSING_CHECK: END\r\n' for data in self._ensure_flush(): yield data def updates(self): """ Handles the UPDATES step of a REPLICATION request. Receives a set of PUT and DELETE subrequests that will be routed to the object server itself for processing. These contain the information requested by the MISSING_CHECK step. The PUT and DELETE subrequests are formatted pretty much exactly like regular HTTP requests, excepting the HTTP version on the first request line. The process is generally: 1. Sender sends `:UPDATES: START` and begins sending the PUT and DELETE subrequests. 2. Receiver gets `:UPDATES: START` and begins routing the subrequests to the object server. 3. Sender sends `:UPDATES: END`. 4. Receiver gets `:UPDATES: END` and sends `:UPDATES: START` and `:UPDATES: END` (assuming no errors). 5. Sender gets `:UPDATES: START` and `:UPDATES: END`. If too many subrequests fail, as configured by replication_failure_threshold and replication_failure_ratio, the receiver will hang up the request early so as to not waste any more time. At step 4, the receiver will send back an error if there were any failures (that didn't cause a hangup due to the above thresholds) so the sender knows the whole was not entirely a success. This is so the sender knows if it can remove an out of place partition, for example. """ with exceptions.MessageTimeout( self.app.client_timeout, 'updates start'): line = self.fp.readline(self.app.network_chunk_size) if line.strip() != ':UPDATES: START': raise Exception('Looking for :UPDATES: START got %r' % line[:1024]) successes = 0 failures = 0 while True: with exceptions.MessageTimeout( self.app.client_timeout, 'updates line'): line = self.fp.readline(self.app.network_chunk_size) if not line or line.strip() == ':UPDATES: END': break # Read first line METHOD PATH of subrequest. method, path = line.strip().split(' ', 1) subreq = swob.Request.blank( '/%s/%s%s' % (self.device, self.partition, path), environ={'REQUEST_METHOD': method}) # Read header lines. content_length = None replication_headers = [] while True: with exceptions.MessageTimeout(self.app.client_timeout): line = self.fp.readline(self.app.network_chunk_size) if not line: raise Exception( 'Got no headers for %s %s' % (method, path)) line = line.strip() if not line: break header, value = line.split(':', 1) header = header.strip().lower() value = value.strip() subreq.headers[header] = value replication_headers.append(header) if header == 'content-length': content_length = int(value) # Establish subrequest body, if needed. if method == 'DELETE': if content_length not in (None, 0): raise Exception( 'DELETE subrequest with content-length %s' % path) elif method == 'PUT': if content_length is None: raise Exception( 'No content-length sent for %s %s' % (method, path)) def subreq_iter(): left = content_length while left > 0: with exceptions.MessageTimeout( self.app.client_timeout, 'updates content'): chunk = self.fp.read( min(left, self.app.network_chunk_size)) if not chunk: raise Exception( 'Early termination for %s %s' % (method, path)) left -= len(chunk) yield chunk subreq.environ['wsgi.input'] = utils.FileLikeIter( subreq_iter()) else: raise Exception('Invalid subrequest method %s' % method) subreq.headers['X-Backend-Replication'] = 'True' if replication_headers: subreq.headers['X-Backend-Replication-Headers'] = \ ' '.join(replication_headers) # Route subrequest and translate response. resp = subreq.get_response(self.app) if http.is_success(resp.status_int) or \ resp.status_int == http.HTTP_NOT_FOUND: successes += 1 else: failures += 1 if failures >= self.app.replication_failure_threshold and ( not successes or float(failures) / successes > self.app.replication_failure_ratio): raise Exception( 'Too many %d failures to %d successes' % (failures, successes)) # The subreq may have failed, but we want to read the rest of the # body from the remote side so we can continue on with the next # subreq. for junk in subreq.environ['wsgi.input']: pass if failures: raise swob.HTTPInternalServerError( 'ERROR: With :UPDATES: %d failures to %d successes' % (failures, successes)) yield ':UPDATES: START\r\n' yield ':UPDATES: END\r\n' for data in self._ensure_flush(): yield data swift-1.13.1/swift/obj/updater.py0000664000175400017540000002431712323703611020020 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import cPickle as pickle import os import signal import sys import time from swift import gettext_ as _ from random import random from eventlet import patcher, Timeout from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.ring import Ring from swift.common.utils import get_logger, renamer, write_pickle, \ dump_recon_cache, config_true_value, ismount from swift.common.daemon import Daemon from swift.obj.diskfile import ASYNCDIR from swift.common.http import is_success, HTTP_NOT_FOUND, \ HTTP_INTERNAL_SERVER_ERROR class ObjectUpdater(Daemon): """Update object information in container listings.""" def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='object-updater') self.devices = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.interval = int(conf.get('interval', 300)) self.container_ring = None self.concurrency = int(conf.get('concurrency', 1)) self.slowdown = float(conf.get('slowdown', 0.01)) self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.successes = 0 self.failures = 0 self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, 'object.recon') def get_container_ring(self): """Get the container ring. Load it, if it hasn't been yet.""" if not self.container_ring: self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring def run_forever(self, *args, **kwargs): """Run the updater continuously.""" time.sleep(random() * self.interval) while True: self.logger.info(_('Begin object update sweep')) begin = time.time() pids = [] # read from container ring to ensure it's fresh self.get_container_ring().get_nodes('') for device in os.listdir(self.devices): if self.mount_check and \ not ismount(os.path.join(self.devices, device)): self.logger.increment('errors') self.logger.warn( _('Skipping %s as it is not mounted'), device) continue while len(pids) >= self.concurrency: pids.remove(os.wait()[0]) pid = os.fork() if pid: pids.append(pid) else: signal.signal(signal.SIGTERM, signal.SIG_DFL) patcher.monkey_patch(all=False, socket=True) self.successes = 0 self.failures = 0 forkbegin = time.time() self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - forkbegin self.logger.info( _('Object update sweep of %(device)s' ' completed: %(elapsed).02fs, %(success)s successes' ', %(fail)s failures'), {'device': device, 'elapsed': elapsed, 'success': self.successes, 'fail': self.failures}) sys.exit() while pids: pids.remove(os.wait()[0]) elapsed = time.time() - begin self.logger.info(_('Object update sweep completed: %.02fs'), elapsed) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) if elapsed < self.interval: time.sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """Run the updater once.""" self.logger.info(_('Begin object update single threaded sweep')) begin = time.time() self.successes = 0 self.failures = 0 for device in os.listdir(self.devices): if self.mount_check and \ not ismount(os.path.join(self.devices, device)): self.logger.increment('errors') self.logger.warn( _('Skipping %s as it is not mounted'), device) continue self.object_sweep(os.path.join(self.devices, device)) elapsed = time.time() - begin self.logger.info( _('Object update single threaded sweep completed: ' '%(elapsed).02fs, %(success)s successes, %(fail)s failures'), {'elapsed': elapsed, 'success': self.successes, 'fail': self.failures}) dump_recon_cache({'object_updater_sweep': elapsed}, self.rcache, self.logger) def object_sweep(self, device): """ If there are async pendings on the device, walk each one and update. :param device: path to device """ start_time = time.time() async_pending = os.path.join(device, ASYNCDIR) if not os.path.isdir(async_pending): return for prefix in os.listdir(async_pending): prefix_path = os.path.join(async_pending, prefix) if not os.path.isdir(prefix_path): continue last_obj_hash = None for update in sorted(os.listdir(prefix_path), reverse=True): update_path = os.path.join(prefix_path, update) if not os.path.isfile(update_path): continue try: obj_hash, timestamp = update.split('-') except ValueError: self.logger.increment('errors') self.logger.error( _('ERROR async pending file with unexpected name %s') % (update_path)) continue if obj_hash == last_obj_hash: self.logger.increment("unlinks") os.unlink(update_path) else: self.process_object_update(update_path, device) last_obj_hash = obj_hash time.sleep(self.slowdown) try: os.rmdir(prefix_path) except OSError: pass self.logger.timing_since('timing', start_time) def process_object_update(self, update_path, device): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device """ try: update = pickle.load(open(update_path, 'rb')) except Exception: self.logger.exception( _('ERROR Pickle problem, quarantining %s'), update_path) self.logger.increment('quarantines') renamer(update_path, os.path.join( device, 'quarantined', 'objects', os.path.basename(update_path))) return successes = update.get('successes', []) part, nodes = self.get_container_ring().get_nodes( update['account'], update['container']) obj = '/%s/%s/%s' % \ (update['account'], update['container'], update['obj']) success = True new_successes = False for node in nodes: if node['id'] not in successes: status = self.object_update(node, part, update['op'], obj, update['headers']) if not is_success(status) and status != HTTP_NOT_FOUND: success = False else: successes.append(node['id']) new_successes = True if success: self.successes += 1 self.logger.increment('successes') self.logger.debug(_('Update sent for %(obj)s %(path)s'), {'obj': obj, 'path': update_path}) self.logger.increment("unlinks") os.unlink(update_path) else: self.failures += 1 self.logger.increment('failures') self.logger.debug(_('Update failed for %(obj)s %(path)s'), {'obj': obj, 'path': update_path}) if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join(device, 'tmp')) def object_update(self, node, part, op, obj, headers): """ Perform the object update to the container :param node: node dictionary from the container ring :param part: partition that holds the container :param op: operation performed (ex: 'POST' or 'DELETE') :param obj: object name being updated :param headers: headers to send with the update """ headers_out = headers.copy() headers_out['user-agent'] = 'obj-updater %s' % os.getpid() try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, op, obj, headers_out) with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() return resp.status except (Exception, Timeout): self.logger.exception(_('ERROR with remote server ' '%(ip)s:%(port)s/%(device)s'), node) return HTTP_INTERNAL_SERVER_ERROR swift-1.13.1/swift/obj/mem_server.py0000664000175400017540000001021712323703611020512 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ In-Memory Object Server for Swift """ import os from swift import gettext_ as _ from eventlet import Timeout from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout from swift.common.http import is_success from swift.obj.mem_diskfile import InMemoryFileSystem from swift.obj import server class ObjectController(server.ObjectController): """ Implements the WSGI application for the Swift In-Memory Object Server. """ def setup(self, conf): """ Nothing specific to do for the in-memory version. :param conf: WSGI configuration parameter """ self._filesystem = InMemoryFileSystem() def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._filesystem.get_diskfile(account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed: %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s'), {'ip': ip, 'port': port, 'dev': contdevice}) # FIXME: For now don't handle async updates def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ pass def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI object server apps""" conf = global_conf.copy() conf.update(local_conf) return ObjectController(conf) swift-1.13.1/swift/obj/expirer.py0000664000175400017540000002317512323703611020033 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import urllib from random import random from time import time from os.path import join from swift import gettext_ as _ import hashlib from eventlet import sleep, Timeout from eventlet.greenpool import GreenPool from swift.common.daemon import Daemon from swift.common.internal_client import InternalClient from swift.common.utils import get_logger, dump_recon_cache from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \ HTTP_PRECONDITION_FAILED class ObjectExpirer(Daemon): """ Daemon that queries the internal hidden expiring_objects_account to discover objects that need to be deleted. :param conf: The daemon configuration. """ def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='object-expirer') self.interval = int(conf.get('interval') or 300) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') conf_path = conf.get('__file__') or '/etc/swift/object-expirer.conf' request_tries = int(conf.get('request_tries') or 3) self.swift = InternalClient(conf_path, 'Swift Object Expirer', request_tries) self.report_interval = int(conf.get('report_interval') or 300) self.report_first_time = self.report_last_time = time() self.report_objects = 0 self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = join(self.recon_cache_path, 'object.recon') self.concurrency = int(conf.get('concurrency', 1)) if self.concurrency < 1: raise ValueError("concurrency must be set to at least 1") self.processes = int(self.conf.get('processes', 0)) self.process = int(self.conf.get('process', 0)) def report(self, final=False): """ Emits a log line report of the progress so far, or the final progress is final=True. :param final: Set to True for the last report once the expiration pass has completed. """ if final: elapsed = time() - self.report_first_time self.logger.info(_('Pass completed in %ds; %d objects expired') % (elapsed, self.report_objects)) dump_recon_cache({'object_expiration_pass': elapsed, 'expired_last_pass': self.report_objects}, self.rcache, self.logger) elif time() - self.report_last_time >= self.report_interval: elapsed = time() - self.report_first_time self.logger.info(_('Pass so far %ds; %d objects expired') % (elapsed, self.report_objects)) self.report_last_time = time() def run_once(self, *args, **kwargs): """ Executes a single pass, looking for objects to expire. :param args: Extra args to fulfill the Daemon interface; this daemon has no additional args. :param kwargs: Extra keyword args to fulfill the Daemon interface; this daemon accepts processes and process keyword args. These will override the values from the config file if provided. """ processes, process = self.get_process_values(kwargs) pool = GreenPool(self.concurrency) containers_to_delete = [] self.report_first_time = self.report_last_time = time() self.report_objects = 0 try: self.logger.debug(_('Run begin')) containers, objects = \ self.swift.get_account_info(self.expiring_objects_account) self.logger.info(_('Pass beginning; %s possible containers; %s ' 'possible objects') % (containers, objects)) for c in self.swift.iter_containers(self.expiring_objects_account): container = c['name'] timestamp = int(container) if timestamp > int(time()): break containers_to_delete.append(container) for o in self.swift.iter_objects(self.expiring_objects_account, container): obj = o['name'].encode('utf8') if processes > 0: obj_process = int( hashlib.md5('%s/%s' % (container, obj)). hexdigest(), 16) if obj_process % processes != process: continue timestamp, actual_obj = obj.split('-', 1) timestamp = int(timestamp) if timestamp > int(time()): break pool.spawn_n( self.delete_object, actual_obj, timestamp, container, obj) pool.waitall() for container in containers_to_delete: try: self.swift.delete_container( self.expiring_objects_account, container, acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT)) except (Exception, Timeout) as err: self.logger.exception( _('Exception while deleting container %s %s') % (container, str(err))) self.logger.debug(_('Run end')) self.report(final=True) except (Exception, Timeout): self.logger.exception(_('Unhandled exception')) def run_forever(self, *args, **kwargs): """ Executes passes forever, looking for objects to expire. :param args: Extra args to fulfill the Daemon interface; this daemon has no additional args. :param kwargs: Extra keyword args to fulfill the Daemon interface; this daemon has no additional keyword args. """ sleep(random() * self.interval) while True: begin = time() try: self.run_once(*args, **kwargs) except (Exception, Timeout): self.logger.exception(_('Unhandled exception')) elapsed = time() - begin if elapsed < self.interval: sleep(random() * (self.interval - elapsed)) def get_process_values(self, kwargs): """ Gets the processes, process from the kwargs if those values exist. Otherwise, return processes, process set in the config file. :param kwargs: Keyword args passed into the run_forever(), run_once() methods. They have values specified on the command line when the daemon is run. """ if kwargs.get('processes') is not None: processes = int(kwargs['processes']) else: processes = self.processes if kwargs.get('process') is not None: process = int(kwargs['process']) else: process = self.process if process < 0: raise ValueError( 'process must be an integer greater than or equal to 0') if processes < 0: raise ValueError( 'processes must be an integer greater than or equal to 0') if processes and process >= processes: raise ValueError( 'process must be less than or equal to processes') return processes, process def delete_object(self, actual_obj, timestamp, container, obj): start_time = time() try: self.delete_actual_object(actual_obj, timestamp) self.swift.delete_object(self.expiring_objects_account, container, obj) self.report_objects += 1 self.logger.increment('objects') except (Exception, Timeout) as err: self.logger.increment('errors') self.logger.exception( _('Exception while deleting object %s %s %s') % (container, obj, str(err))) self.logger.timing_since('timing', start_time) self.report() def delete_actual_object(self, actual_obj, timestamp): """ Deletes the end-user object indicated by the actual object name given '//' if and only if the X-Delete-At value of the object is exactly the timestamp given. :param actual_obj: The name of the end-user object to delete: '//' :param timestamp: The timestamp the X-Delete-At value must match to perform the actual delete. """ path = '/v1/' + urllib.quote(actual_obj.lstrip('/')) self.swift.make_request('DELETE', path, {'X-If-Delete-At': str(timestamp)}, (2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED)) swift-1.13.1/swift/obj/server.py0000664000175400017540000010245612323703614017666 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Object Server for Swift """ import cPickle as pickle import os import multiprocessing import time import traceback import socket import math from datetime import datetime from swift import gettext_ as _ from hashlib import md5 from eventlet import sleep, Timeout from swift.common.utils import public, get_logger, \ config_true_value, timing_stats, replication, normalize_delete_at_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ check_float, check_utf8 from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \ DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout from swift.obj import ssync_receiver from swift.common.http import is_success from swift.common.request_helpers import split_and_validate_path, is_user_meta from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \ HTTPInsufficientStorage, HTTPForbidden, HTTPException, HeaderKeyDict, \ HTTPConflict from swift.obj.diskfile import DATAFILE_SYSTEM_META, DiskFileManager class ObjectController(object): """Implements the WSGI application for the Swift Object Server.""" def __init__(self, conf, logger=None): """ Creates a new WSGI application for the Swift Object Server. An example configuration is given at /etc/object-server.conf-sample or /etc/swift/object-server.conf-sample. """ self.logger = logger or get_logger(conf, log_route='object-server') self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.client_timeout = int(conf.get('client_timeout', 60)) self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.network_chunk_size = int(conf.get('network_chunk_size', 65536)) self.log_requests = config_true_value(conf.get('log_requests', 'true')) self.max_upload_time = int(conf.get('max_upload_time', 86400)) self.slow = int(conf.get('slow', 0)) self.keep_cache_private = \ config_true_value(conf.get('keep_cache_private', 'false')) replication_server = conf.get('replication_server', None) if replication_server is not None: replication_server = config_true_value(replication_server) self.replication_server = replication_server default_allowed_headers = ''' content-disposition, content-encoding, x-delete-at, x-object-manifest, x-static-large-object, ''' extra_allowed_headers = [ header.strip().lower() for header in conf.get( 'allowed_headers', default_allowed_headers).split(',') if header.strip() ] self.allowed_headers = set() for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) self.expiring_objects_account = \ (conf.get('auto_create_account_prefix') or '.') + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) # Initialization was successful, so now apply the network chunk size # parameter as the default read / write buffer size for the network # sockets. # # NOTE WELL: This is a class setting, so until we get set this on a # per-connection basis, this affects reading and writing on ALL # sockets, those between the proxy servers and external clients, and # those between the proxy servers and the other internal servers. # # ** Because the primary motivation for this is to optimize how data # is written back to the proxy server, we could use the value from the # disk_chunk_size parameter. However, it affects all created sockets # using this class so we have chosen to tie it to the # network_chunk_size parameter value instead. socket._fileobject.default_bufsize = self.network_chunk_size # Provide further setup sepecific to an object server implemenation. self.setup(conf) def setup(self, conf): """ Implementation specific setup. This method is called at the very end by the constructor to allow a specific implementation to modify existing attributes or add its own attributes. :param conf: WSGI configuration parameter """ # Common on-disk hierarchy shared across account, container and object # servers. self._diskfile_mgr = DiskFileManager(conf, self.logger) # This is populated by global_conf_callback way below as the semaphore # is shared by all workers. if 'replication_semaphore' in conf: # The value was put in a list so it could get past paste self.replication_semaphore = conf['replication_semaphore'][0] else: self.replication_semaphore = None self.replication_failure_threshold = int( conf.get('replication_failure_threshold') or 100) self.replication_failure_ratio = float( conf.get('replication_failure_ratio') or 1.0) def get_diskfile(self, device, partition, account, container, obj, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. An implementation of the object server that wants to use a different DiskFile class would simply over-ride this method to provide that behavior. """ return self._diskfile_mgr.get_diskfile( device, partition, account, container, obj, **kwargs) def async_update(self, op, account, container, obj, host, partition, contdevice, headers_out, objdevice): """ Sends or saves an async update. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param host: host that the container is on :param partition: partition that the container is on :param contdevice: device name that the container is on :param headers_out: dictionary of headers to send in the container request :param objdevice: device name that the object is in """ headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: with ConnectionTimeout(self.conn_timeout): ip, port = host.rsplit(':', 1) conn = http_connect(ip, port, contdevice, partition, op, full_path, headers_out) with Timeout(self.node_timeout): response = conn.getresponse() response.read() if is_success(response.status): return else: self.logger.error(_( 'ERROR Container update failed ' '(saving for async update later): %(status)d ' 'response from %(ip)s:%(port)s/%(dev)s'), {'status': response.status, 'ip': ip, 'port': port, 'dev': contdevice}) except (Exception, Timeout): self.logger.exception(_( 'ERROR container update failed with ' '%(ip)s:%(port)s/%(dev)s (saving for async update later)'), {'ip': ip, 'port': port, 'dev': contdevice}) data = {'op': op, 'account': account, 'container': container, 'obj': obj, 'headers': headers_out} timestamp = headers_out['x-timestamp'] self._diskfile_mgr.pickle_async_update(objdevice, account, container, obj, data, timestamp) def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ headers_in = request.headers conthosts = [h.strip() for h in headers_in.get('X-Container-Host', '').split(',')] contdevices = [d.strip() for d in headers_in.get('X-Container-Device', '').split(',')] contpartition = headers_in.get('X-Container-Partition', '') if len(conthosts) != len(contdevices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error(_('ERROR Container update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"') % (headers_in.get('X-Container-Host', ''), headers_in.get('X-Container-Device', ''))) return if contpartition: updates = zip(conthosts, contdevices) else: updates = [] headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') headers_out['referer'] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, request, objdevice): """ Update the expiring objects container when objects are updated. :param op: operation performed (ex: 'PUT', or 'DELETE') :param delete_at: scheduled delete in UNIX seconds, int :param account: account name for the object :param container: container name for the object :param obj: object name :param request: the original request driving the update :param objdevice: device name that the object is in """ if config_true_value( request.headers.get('x-backend-replication', 'f')): return delete_at = normalize_delete_at_timestamp(delete_at) updates = [(None, None)] partition = None hosts = contdevices = [None] headers_in = request.headers headers_out = HeaderKeyDict({ 'x-timestamp': headers_in['x-timestamp'], 'x-trans-id': headers_in.get('x-trans-id', '-'), 'referer': request.as_referer()}) if op != 'DELETE': delete_at_container = headers_in.get('X-Delete-At-Container', None) if not delete_at_container: self.logger.warning( 'X-Delete-At-Container header must be specified for ' 'expiring objects background %s to work properly. Making ' 'best guess as to the container name for now.' % op) # TODO(gholt): In a future release, change the above warning to # a raised exception and remove the guess code below. delete_at_container = ( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') contdevices = headers_in.get('X-Delete-At-Device', '') updates = [upd for upd in zip((h.strip() for h in hosts.split(',')), (c.strip() for c in contdevices.split(','))) if all(upd) and partition] if not updates: updates = [(None, None)] headers_out['x-size'] = '0' headers_out['x-content-type'] = 'text/plain' headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e' else: # DELETEs of old expiration data have no way of knowing what the # old X-Delete-At-Container was at the time of the initial setting # of the data, so a best guess is made here. # Worst case is a DELETE is issued now for something that doesn't # exist there and the original data is left where it is, where # it will be ignored when the expirer eventually tries to issue the # object DELETE later since the X-Delete-At value won't match up. delete_at_container = str( int(delete_at) / self.expiring_objects_container_divisor * self.expiring_objects_container_divisor) delete_at_container = normalize_delete_at_timestamp( delete_at_container) for host, contdevice in updates: self.async_update( op, self.expiring_objects_account, delete_at_container, '%s-%s/%s/%s' % (delete_at, account, container, obj), host, partition, contdevice, headers_out, objdevice) @public @timing_stats() def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request) @public @timing_stats() def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device) return HTTPCreated(request=request, etag=etag) @public @timing_stats() def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if_unmodified_since = request.if_unmodified_since if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) if_modified_since = request.if_modified_since if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) return resp @public @timing_stats(sample_rate=0.8) def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = metadata['X-Timestamp'] response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response @public @timing_stats() def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileExpired as e: orig_timestamp = e.timestamp orig_metadata = e.metadata response_class = HTTPNotFound except DiskFileDeleted as e: orig_timestamp = e.timestamp orig_metadata = {} response_class = HTTPNotFound except (DiskFileNotExist, DiskFileQuarantined): orig_timestamp = 0 orig_metadata = {} response_class = HTTPNotFound else: orig_timestamp = orig_metadata.get('X-Timestamp', 0) if orig_timestamp < request.headers['x-timestamp']: response_class = HTTPNoContent else: response_class = HTTPConflict orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) try: req_if_delete_at_val = request.headers['x-if-delete-at'] req_if_delete_at = int(req_if_delete_at_val) except KeyError: pass except ValueError: return HTTPBadRequest( request=request, body='Bad X-If-Delete-At header value') else: if orig_delete_at != req_if_delete_at: return HTTPPreconditionFailed( request=request, body='X-If-Delete-At and X-Delete-At do not match') if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) req_timestamp = request.headers['X-Timestamp'] if orig_timestamp < req_timestamp: disk_file.delete(req_timestamp) self.container_update( 'DELETE', account, container, obj, request, HeaderKeyDict({'x-timestamp': req_timestamp}), device) return response_class(request=request) @public @replication @timing_stats(sample_rate=0.1) def REPLICATE(self, request): """ Handle REPLICATE requests for the Swift Object Server. This is used by the object replicator to get hashes for directories. """ device, partition, suffix = split_and_validate_path( request, 2, 3, True) try: hashes = self._diskfile_mgr.get_hashes(device, partition, suffix) except DiskFileDeviceUnavailable: resp = HTTPInsufficientStorage(drive=device, request=request) else: resp = Response(body=pickle.dumps(hashes)) return resp @public @replication @timing_stats(sample_rate=0.1) def REPLICATION(self, request): return Response(app_iter=ssync_receiver.Receiver(self, request)()) def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', req.headers.get('x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response) def global_conf_callback(preloaded_app_conf, global_conf): """ Callback for swift.common.wsgi.run_wsgi during the global_conf creation so that we can add our replication_semaphore, used to limit the number of concurrent REPLICATION_REQUESTS across all workers. :param preloaded_app_conf: The preloaded conf for the WSGI app. This conf instance will go away, so just read from it, don't write. :param global_conf: The global conf that will eventually be passed to the app_factory function later. This conf is created before the worker subprocesses are forked, so can be useful to set up semaphores, shared memory, etc. """ replication_concurrency = int( preloaded_app_conf.get('replication_concurrency') or 4) if replication_concurrency: # Have to put the value in a list so it can get past paste global_conf['replication_semaphore'] = [ multiprocessing.BoundedSemaphore(replication_concurrency)] def app_factory(global_conf, **local_conf): """paste.deploy app factory for creating WSGI object server apps""" conf = global_conf.copy() conf.update(local_conf) return ObjectController(conf) swift-1.13.1/swift/obj/auditor.py0000664000175400017540000003204512323703611020020 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import time import signal from swift import gettext_ as _ from contextlib import closing from eventlet import Timeout from swift.obj import diskfile from swift.common.utils import get_logger, ratelimit_sleep, dump_recon_cache, \ list_from_csv, json, listdir from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist from swift.common.daemon import Daemon SLEEP_BETWEEN_AUDITS = 30 class AuditorWorker(object): """Walk through file system to audit objects""" def __init__(self, conf, logger, rcache, devices, zero_byte_only_at_fps=0): self.conf = conf self.logger = logger self.devices = devices self.diskfile_mgr = diskfile.DiskFileManager(conf, self.logger) self.max_files_per_second = float(conf.get('files_per_second', 20)) self.max_bytes_per_second = float(conf.get('bytes_per_second', 10000000)) self.auditor_type = 'ALL' self.zero_byte_only_at_fps = zero_byte_only_at_fps if self.zero_byte_only_at_fps: self.max_files_per_second = float(self.zero_byte_only_at_fps) self.auditor_type = 'ZBF' self.log_time = int(conf.get('log_time', 3600)) self.files_running_time = 0 self.bytes_running_time = 0 self.bytes_processed = 0 self.total_bytes_processed = 0 self.total_files_processed = 0 self.passes = 0 self.quarantines = 0 self.errors = 0 self.rcache = rcache self.stats_sizes = sorted( [int(s) for s in list_from_csv(conf.get('object_size_stats'))]) self.stats_buckets = dict( [(s, 0) for s in self.stats_sizes + ['OVER']]) def create_recon_nested_dict(self, top_level_key, device_list, item): if device_list: device_key = ''.join(sorted(device_list)) return {top_level_key: {device_key: item}} else: return {top_level_key: item} def audit_all_objects(self, mode='once', device_dirs=None): description = '' if device_dirs: device_dir_str = ','.join(sorted(device_dirs)) description = _(' - %s') % device_dir_str self.logger.info(_('Begin object audit "%s" mode (%s%s)') % (mode, self.auditor_type, description)) begin = reported = time.time() self.total_bytes_processed = 0 self.total_files_processed = 0 total_quarantines = 0 total_errors = 0 time_auditing = 0 all_locs = self.diskfile_mgr.object_audit_location_generator( device_dirs=device_dirs) for location in all_locs: loop_time = time.time() self.failsafe_object_audit(location) self.logger.timing_since('timing', loop_time) self.files_running_time = ratelimit_sleep( self.files_running_time, self.max_files_per_second) self.total_files_processed += 1 now = time.time() if now - reported >= self.log_time: self.logger.info(_( 'Object audit (%(type)s). ' 'Since %(start_time)s: Locally: %(passes)d passed, ' '%(quars)d quarantined, %(errors)d errors ' 'files/sec: %(frate).2f , bytes/sec: %(brate).2f, ' 'Total time: %(total).2f, Auditing time: %(audit).2f, ' 'Rate: %(audit_rate).2f') % { 'type': '%s%s' % (self.auditor_type, description), 'start_time': time.ctime(reported), 'passes': self.passes, 'quars': self.quarantines, 'errors': self.errors, 'frate': self.passes / (now - reported), 'brate': self.bytes_processed / (now - reported), 'total': (now - begin), 'audit': time_auditing, 'audit_rate': time_auditing / (now - begin)}) cache_entry = self.create_recon_nested_dict( 'object_auditor_stats_%s' % (self.auditor_type), device_dirs, {'errors': self.errors, 'passes': self.passes, 'quarantined': self.quarantines, 'bytes_processed': self.bytes_processed, 'start_time': reported, 'audit_time': time_auditing}) dump_recon_cache(cache_entry, self.rcache, self.logger) reported = now total_quarantines += self.quarantines total_errors += self.errors self.passes = 0 self.quarantines = 0 self.errors = 0 self.bytes_processed = 0 time_auditing += (now - loop_time) # Avoid divide by zero during very short runs elapsed = (time.time() - begin) or 0.000001 self.logger.info(_( 'Object audit (%(type)s) "%(mode)s" mode ' 'completed: %(elapsed).02fs. Total quarantined: %(quars)d, ' 'Total errors: %(errors)d, Total files/sec: %(frate).2f, ' 'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, ' 'Rate: %(audit_rate).2f') % { 'type': '%s%s' % (self.auditor_type, description), 'mode': mode, 'elapsed': elapsed, 'quars': total_quarantines + self.quarantines, 'errors': total_errors + self.errors, 'frate': self.total_files_processed / elapsed, 'brate': self.total_bytes_processed / elapsed, 'audit': time_auditing, 'audit_rate': time_auditing / elapsed}) # Clear recon cache entry if device_dirs is set if device_dirs: cache_entry = self.create_recon_nested_dict( 'object_auditor_stats_%s' % (self.auditor_type), device_dirs, {}) dump_recon_cache(cache_entry, self.rcache, self.logger) if self.stats_sizes: self.logger.info( _('Object audit stats: %s') % json.dumps(self.stats_buckets)) def record_stats(self, obj_size): """ Based on config's object_size_stats will keep track of how many objects fall into the specified ranges. For example with the following: object_size_stats = 10, 100, 1024 and your system has 3 objects of sizes: 5, 20, and 10000 bytes the log will look like: {"10": 1, "100": 1, "1024": 0, "OVER": 1} """ for size in self.stats_sizes: if obj_size <= size: self.stats_buckets[size] += 1 break else: self.stats_buckets["OVER"] += 1 def failsafe_object_audit(self, location): """ Entrypoint to object_audit, with a failsafe generic exception handler. """ try: self.object_audit(location) except (Exception, Timeout): self.logger.increment('errors') self.errors += 1 self.logger.exception(_('ERROR Trying to audit %s'), location) def object_audit(self, location): """ Audits the given object location. :param location: an audit location (from diskfile.object_audit_location_generator) """ def raise_dfq(msg): raise DiskFileQuarantined(msg) try: df = self.diskfile_mgr.get_diskfile_from_audit_location(location) with df.open(): metadata = df.get_metadata() obj_size = int(metadata['Content-Length']) if self.stats_sizes: self.record_stats(obj_size) if self.zero_byte_only_at_fps and obj_size: self.passes += 1 return reader = df.reader(_quarantine_hook=raise_dfq) with closing(reader): for chunk in reader: chunk_len = len(chunk) self.bytes_running_time = ratelimit_sleep( self.bytes_running_time, self.max_bytes_per_second, incr_by=chunk_len) self.bytes_processed += chunk_len self.total_bytes_processed += chunk_len except DiskFileNotExist: return except DiskFileQuarantined as err: self.quarantines += 1 self.logger.error(_('ERROR Object %(obj)s failed audit and was' ' quarantined: %(err)s'), {'obj': location, 'err': err}) self.passes += 1 class ObjectAuditor(Daemon): """Audit objects.""" def __init__(self, conf, **options): self.conf = conf self.logger = get_logger(conf, log_route='object-auditor') self.devices = conf.get('devices', '/srv/node') self.conf_zero_byte_fps = int( conf.get('zero_byte_files_per_second', 50)) self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") def _sleep(self): time.sleep(SLEEP_BETWEEN_AUDITS) def clear_recon_cache(self, auditor_type): """Clear recon cache entries""" dump_recon_cache({'object_auditor_stats_%s' % auditor_type: {}}, self.rcache, self.logger) def run_audit(self, **kwargs): """Run the object audit""" mode = kwargs.get('mode') zero_byte_only_at_fps = kwargs.get('zero_byte_fps', 0) device_dirs = kwargs.get('device_dirs') worker = AuditorWorker(self.conf, self.logger, self.rcache, self.devices, zero_byte_only_at_fps=zero_byte_only_at_fps) worker.audit_all_objects(mode=mode, device_dirs=device_dirs) def fork_child(self, zero_byte_fps=False, **kwargs): """Child execution""" pid = os.fork() if pid: return pid else: signal.signal(signal.SIGTERM, signal.SIG_DFL) if zero_byte_fps: kwargs['zero_byte_fps'] = self.conf_zero_byte_fps self.run_audit(**kwargs) sys.exit() def audit_loop(self, parent, zbo_fps, override_devices=None, **kwargs): """Audit loop""" self.clear_recon_cache('ALL') self.clear_recon_cache('ZBF') kwargs['device_dirs'] = override_devices if parent: kwargs['zero_byte_fps'] = zbo_fps self.run_audit(**kwargs) else: pids = [] if self.conf_zero_byte_fps: zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.append(zbf_pid) pids.append(self.fork_child(**kwargs)) while pids: pid = os.wait()[0] # ZBF scanner must be restarted as soon as it finishes if self.conf_zero_byte_fps and pid == zbf_pid and \ len(pids) > 1: kwargs['device_dirs'] = override_devices zbf_pid = self.fork_child(zero_byte_fps=True, **kwargs) pids.append(zbf_pid) pids.remove(pid) def run_forever(self, *args, **kwargs): """Run the object audit until stopped.""" # zero byte only command line option zbo_fps = kwargs.get('zero_byte_fps', 0) parent = False if zbo_fps: # only start parent parent = True kwargs = {'mode': 'forever'} while True: try: self.audit_loop(parent, zbo_fps, **kwargs) except (Exception, Timeout): self.logger.exception(_('ERROR auditing')) self._sleep() def run_once(self, *args, **kwargs): """Run the object audit once""" # zero byte only command line option zbo_fps = kwargs.get('zero_byte_fps', 0) override_devices = list_from_csv(kwargs.get('devices')) # Remove bogus entries and duplicates from override_devices override_devices = list( set(listdir(self.devices)).intersection(set(override_devices))) parent = False if zbo_fps: # only start parent parent = True kwargs = {'mode': 'once'} try: self.audit_loop(parent, zbo_fps, override_devices=override_devices, **kwargs) except (Exception, Timeout): self.logger.exception(_('ERROR auditing')) swift-1.13.1/swift/obj/diskfile.py0000664000175400017540000015055512323703611020152 0ustar jenkinsjenkins00000000000000# Copyright (c) 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ Disk File Interface for the Swift Object Server The `DiskFile`, `DiskFileWriter` and `DiskFileReader` classes combined define the on-disk abstraction layer for supporting the object server REST API interfaces (excluding `REPLICATE`). Other implementations wishing to provide an alternative backend for the object server must implement the three classes. An example alternative implementation can be found in the `mem_server.py` and `mem_diskfile.py` modules along size this one. The `DiskFileManager` is a reference implemenation specific class and is not part of the backend API. The remaining methods in this module are considered implementation specifc and are also not considered part of the backend API. """ import cPickle as pickle import errno import os import time import uuid import hashlib import logging import traceback from os.path import basename, dirname, exists, getmtime, join from random import shuffle from tempfile import mkstemp from contextlib import contextmanager from collections import defaultdict from xattr import getxattr, setxattr from eventlet import Timeout from swift import gettext_ as _ from swift.common.constraints import check_mount from swift.common.utils import mkdirs, normalize_timestamp, \ storage_directory, hash_path, renamer, fallocate, fsync, \ fdatasync, drop_buffer_cache, ThreadPool, lock_path, write_pickle, \ config_true_value, listdir, split_path, ismount, remove_file from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \ DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \ DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \ ReplicationLockTimeout, DiskFileExpired from swift.common.swob import multi_range_iterator PICKLE_PROTOCOL = 2 ONE_WEEK = 604800 HASH_FILE = 'hashes.pkl' METADATA_KEY = 'user.swift.metadata' # These are system-set metadata keys that cannot be changed with a POST. # They should be lowercase. DATAFILE_SYSTEM_META = set('content-length content-type deleted etag'.split()) DATADIR = 'objects' ASYNCDIR = 'async_pending' def read_metadata(fd): """ Helper function to read the pickled metadata from an object file. :param fd: file descriptor or filename to load the metadata from :returns: dictionary of metadata """ metadata = '' key = 0 try: while True: metadata += getxattr(fd, '%s%s' % (METADATA_KEY, (key or ''))) key += 1 except IOError: pass return pickle.loads(metadata) def write_metadata(fd, metadata): """ Helper function to write pickled metadata for an object file. :param fd: file descriptor or filename to write the metadata :param metadata: metadata to write """ metastr = pickle.dumps(metadata, PICKLE_PROTOCOL) key = 0 while metastr: setxattr(fd, '%s%s' % (METADATA_KEY, key or ''), metastr[:254]) metastr = metastr[254:] key += 1 def quarantine_renamer(device_path, corrupted_file_path): """ In the case that a file is corrupted, move it to a quarantined area to allow replication to fix it. :params device_path: The path to the device the corrupted file is on. :params corrupted_file_path: The path to the file you want quarantined. :returns: path (str) of directory the file was moved to :raises OSError: re-raises non errno.EEXIST / errno.ENOTEMPTY exceptions from rename """ from_dir = dirname(corrupted_file_path) to_dir = join(device_path, 'quarantined', 'objects', basename(from_dir)) invalidate_hash(dirname(from_dir)) try: renamer(from_dir, to_dir) except OSError as e: if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): raise to_dir = "%s-%s" % (to_dir, uuid.uuid4().hex) renamer(from_dir, to_dir) return to_dir def get_ondisk_files(files, datadir): """ Given a simple list of files names, determine the files to use. :params files: simple set of files as a python list :params datadir: directory name files are from for convenience :returns: a tuple of data, meta and ts (tombstone) files, in one of two states: * ts_file is not None, data_file is None, meta_file is None object is considered deleted * data_file is not None, ts_file is None object exists, and optionally has fast-POST metadata """ files.sort(reverse=True) data_file = meta_file = ts_file = None for afile in files: assert ts_file is None, "On-disk file search loop" \ " continuing after tombstone, %s, encountered" % ts_file assert data_file is None, "On-disk file search loop" \ " continuing after data file, %s, encountered" % data_file if afile.endswith('.ts'): meta_file = None ts_file = join(datadir, afile) break if afile.endswith('.meta') and not meta_file: meta_file = join(datadir, afile) # NOTE: this does not exit this loop, since a fast-POST # operation just updates metadata, writing one or more # .meta files, the data file will have an older timestamp, # so we keep looking. continue if afile.endswith('.data'): data_file = join(datadir, afile) break assert ((data_file is None and meta_file is None and ts_file is None) or (ts_file is not None and data_file is None and meta_file is None) or (data_file is not None and ts_file is None)), \ "On-disk file search algorithm contract is broken: data_file:" \ " %s, meta_file: %s, ts_file: %s" % (data_file, meta_file, ts_file) return data_file, meta_file, ts_file def hash_cleanup_listdir(hsh_path, reclaim_age=ONE_WEEK): """ List contents of a hash directory and clean up any old files. :param hsh_path: object hash path :param reclaim_age: age in seconds at which to remove tombstones :returns: list of files remaining in the directory, reverse sorted """ files = listdir(hsh_path) if len(files) == 1: if files[0].endswith('.ts'): # remove tombstones older than reclaim_age ts = files[0].rsplit('.', 1)[0] if (time.time() - float(ts)) > reclaim_age: remove_file(join(hsh_path, files[0])) files.remove(files[0]) elif files: files.sort(reverse=True) data_file, meta_file, ts_file = get_ondisk_files(files, '') newest_file = data_file or ts_file for filename in list(files): if ((filename < newest_file) or (meta_file and filename.endswith('.meta') and filename < meta_file)): remove_file(join(hsh_path, filename)) files.remove(filename) return files def hash_suffix(path, reclaim_age): """ Performs reclamation and returns an md5 of all (remaining) files. :param reclaim_age: age in seconds at which to remove tombstones :raises PathNotDir: if given path is not a valid directory :raises OSError: for non-ENOTDIR errors """ md5 = hashlib.md5() try: path_contents = sorted(os.listdir(path)) except OSError as err: if err.errno in (errno.ENOTDIR, errno.ENOENT): raise PathNotDir() raise for hsh in path_contents: hsh_path = join(path, hsh) try: files = hash_cleanup_listdir(hsh_path, reclaim_age) except OSError as err: if err.errno == errno.ENOTDIR: partition_path = dirname(path) objects_path = dirname(partition_path) device_path = dirname(objects_path) quar_path = quarantine_renamer(device_path, hsh_path) logging.exception( _('Quarantined %s to %s because it is not a directory') % (hsh_path, quar_path)) continue raise if not files: try: os.rmdir(hsh_path) except OSError: pass for filename in files: md5.update(filename) try: os.rmdir(path) except OSError: pass return md5.hexdigest() def invalidate_hash(suffix_dir): """ Invalidates the hash for a suffix_dir in the partition's hashes file. :param suffix_dir: absolute path to suffix dir whose hash needs invalidating """ suffix = basename(suffix_dir) partition_dir = dirname(suffix_dir) hashes_file = join(partition_dir, HASH_FILE) with lock_path(partition_dir): try: with open(hashes_file, 'rb') as fp: hashes = pickle.load(fp) if suffix in hashes and not hashes[suffix]: return except Exception: return hashes[suffix] = None write_pickle(hashes, hashes_file, partition_dir, PICKLE_PROTOCOL) def get_hashes(partition_dir, recalculate=None, do_listdir=False, reclaim_age=ONE_WEEK): """ Get a list of hashes for the suffix dir. do_listdir causes it to mistrust the hash cache for suffix existence at the (unexpectedly high) cost of a listdir. reclaim_age is just passed on to hash_suffix. :param partition_dir: absolute path of partition to get hashes for :param recalculate: list of suffixes which should be recalculated when got :param do_listdir: force existence check for all hashes in the partition :param reclaim_age: age at which to remove tombstones :returns: tuple of (number of suffix dirs hashed, dictionary of hashes) """ hashed = 0 hashes_file = join(partition_dir, HASH_FILE) modified = False force_rewrite = False hashes = {} mtime = -1 if recalculate is None: recalculate = [] try: with open(hashes_file, 'rb') as fp: hashes = pickle.load(fp) mtime = getmtime(hashes_file) except Exception: do_listdir = True force_rewrite = True if do_listdir: for suff in os.listdir(partition_dir): if len(suff) == 3: hashes.setdefault(suff, None) modified = True hashes.update((hash_, None) for hash_ in recalculate) for suffix, hash_ in hashes.items(): if not hash_: suffix_dir = join(partition_dir, suffix) try: hashes[suffix] = hash_suffix(suffix_dir, reclaim_age) hashed += 1 except PathNotDir: del hashes[suffix] except OSError: logging.exception(_('Error hashing suffix')) modified = True if modified: with lock_path(partition_dir): if force_rewrite or not exists(hashes_file) or \ getmtime(hashes_file) == mtime: write_pickle( hashes, hashes_file, partition_dir, PICKLE_PROTOCOL) return hashed, hashes return get_hashes(partition_dir, recalculate, do_listdir, reclaim_age) else: return hashed, hashes class AuditLocation(object): """ Represents an object location to be audited. Other than being a bucket of data, the only useful thing this does is stringify to a filesystem path so the auditor's logs look okay. """ def __init__(self, path, device, partition): self.path, self.device, self.partition = path, device, partition def __str__(self): return str(self.path) def object_audit_location_generator(devices, mount_check=True, logger=None, device_dirs=None): """ Given a devices path (e.g. "/srv/node"), yield an AuditLocation for all objects stored under that directory if device_dirs isn't set. If device_dirs is set, only yield AuditLocation for the objects under the entries in device_dirs. The AuditLocation only knows the path to the hash directory, not to the .data file therein (if any). This is to avoid a double listdir(hash_dir); the DiskFile object will always do one, so we don't. :param devices: parent directory of the devices to be audited :param mount_check: flag to check if a mount check should be performed on devices :param logger: a logger object :device_dirs: a list of directories under devices to traverse """ if not device_dirs: device_dirs = listdir(devices) else: # remove bogus devices and duplicates from device_dirs device_dirs = list( set(listdir(devices)).intersection(set(device_dirs))) # randomize devices in case of process restart before sweep completed shuffle(device_dirs) for device in device_dirs: if mount_check and not \ ismount(os.path.join(devices, device)): if logger: logger.debug( _('Skipping %s as it is not mounted'), device) continue datadir_path = os.path.join(devices, device, DATADIR) partitions = listdir(datadir_path) for partition in partitions: part_path = os.path.join(datadir_path, partition) try: suffixes = listdir(part_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for asuffix in suffixes: suff_path = os.path.join(part_path, asuffix) try: hashes = listdir(suff_path) except OSError as e: if e.errno != errno.ENOTDIR: raise continue for hsh in hashes: hsh_path = os.path.join(suff_path, hsh) yield AuditLocation(hsh_path, device, partition) class DiskFileManager(object): """ Management class for devices, providing common place for shared parameters and methods not provided by the DiskFile class (which primarily services the object server REST API layer). The `get_diskfile()` method is how this implementation creates a `DiskFile` object. .. note:: This class is reference implementation specific and not part of the pluggable on-disk backend API. .. note:: TODO(portante): Not sure what the right name to recommend here, as "manager" seemed generic enough, though suggestions are welcome. :param conf: caller provided configuration object :param logger: caller provided logger """ def __init__(self, conf, logger): self.logger = logger self.devices = conf.get('devices', '/srv/node') self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536)) self.keep_cache_size = int(conf.get('keep_cache_size', 5242880)) self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024 self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.reclaim_age = int(conf.get('reclaim_age', ONE_WEEK)) self.replication_one_per_device = config_true_value( conf.get('replication_one_per_device', 'true')) self.replication_lock_timeout = int(conf.get( 'replication_lock_timeout', 15)) threads_per_disk = int(conf.get('threads_per_disk', '0')) self.threadpools = defaultdict( lambda: ThreadPool(nthreads=threads_per_disk)) def construct_dev_path(self, device): """ Construct the path to a device without checking if it is mounted. :param device: name of target device :returns: full path to the device """ return os.path.join(self.devices, device) def get_dev_path(self, device, mount_check=None): """ Return the path to a device, checking to see that it is a proper mount point based on a configuration parameter. :param device: name of target device :param mount_check: whether or not to check mountedness of device. Defaults to bool(self.mount_check). :returns: full path to the device, None if the path to the device is not a proper mount point. """ should_check = self.mount_check if mount_check is None else mount_check if should_check and not check_mount(self.devices, device): dev_path = None else: dev_path = os.path.join(self.devices, device) return dev_path @contextmanager def replication_lock(self, device): """ A context manager that will lock on the device given, if configured to do so. :raises ReplicationLockTimeout: If the lock on the device cannot be granted within the configured timeout. """ if self.replication_one_per_device: dev_path = self.get_dev_path(device) with lock_path( dev_path, timeout=self.replication_lock_timeout, timeout_class=ReplicationLockTimeout): yield True else: yield True def pickle_async_update(self, device, account, container, obj, data, timestamp): device_path = self.construct_dev_path(device) async_dir = os.path.join(device_path, ASYNCDIR) ohash = hash_path(account, container, obj) self.threadpools[device].run_in_thread( write_pickle, data, os.path.join(async_dir, ohash[-3:], ohash + '-' + normalize_timestamp(timestamp)), os.path.join(device_path, 'tmp')) self.logger.increment('async_pendings') def get_diskfile(self, device, partition, account, container, obj, **kwargs): dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() return DiskFile(self, dev_path, self.threadpools[device], partition, account, container, obj, **kwargs) def object_audit_location_generator(self, device_dirs=None): return object_audit_location_generator(self.devices, self.mount_check, self.logger, device_dirs) def get_diskfile_from_audit_location(self, audit_location): dev_path = self.get_dev_path(audit_location.device, mount_check=False) return DiskFile.from_hash_dir( self, audit_location.path, dev_path, audit_location.partition) def get_diskfile_from_hash(self, device, partition, object_hash, **kwargs): """ Returns a DiskFile instance for an object at the given object_hash. Just in case someone thinks of refactoring, be sure DiskFileDeleted is *not* raised, but the DiskFile instance representing the tombstoned object is returned instead. :raises DiskFileNotExist: if the object does not exist """ dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() object_path = os.path.join( dev_path, DATADIR, partition, object_hash[-3:], object_hash) try: filenames = hash_cleanup_listdir(object_path, self.reclaim_age) except OSError as err: if err.errno == errno.ENOTDIR: quar_path = quarantine_renamer(dev_path, object_path) logging.exception( _('Quarantined %s to %s because it is not a ' 'directory') % (object_path, quar_path)) raise DiskFileNotExist() if err.errno != errno.ENOENT: raise raise DiskFileNotExist() if not filenames: raise DiskFileNotExist() try: metadata = read_metadata(os.path.join(object_path, filenames[-1])) except EOFError: raise DiskFileNotExist() try: account, container, obj = split_path( metadata.get('name', ''), 3, 3, True) except ValueError: raise DiskFileNotExist() return DiskFile(self, dev_path, self.threadpools[device], partition, account, container, obj, **kwargs) def get_hashes(self, device, partition, suffix): dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() partition_path = os.path.join(dev_path, DATADIR, partition) if not os.path.exists(partition_path): mkdirs(partition_path) suffixes = suffix.split('-') if suffix else [] _junk, hashes = self.threadpools[device].force_run_in_thread( get_hashes, partition_path, recalculate=suffixes) return hashes def _listdir(self, path): try: return os.listdir(path) except OSError as err: if err.errno != errno.ENOENT: self.logger.error( 'ERROR: Skipping %r due to error with listdir attempt: %s', path, err) return [] def yield_suffixes(self, device, partition): """ Yields tuples of (full_path, suffix_only) for suffixes stored on the given device and partition. """ dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() partition_path = os.path.join(dev_path, DATADIR, partition) for suffix in self._listdir(partition_path): if len(suffix) != 3: continue try: int(suffix, 16) except ValueError: continue yield (os.path.join(partition_path, suffix), suffix) def yield_hashes(self, device, partition, suffixes=None): """ Yields tuples of (full_path, hash_only, timestamp) for object information stored for the given device, partition, and (optionally) suffixes. If suffixes is None, all stored suffixes will be searched for object hashes. Note that if suffixes is not None but empty, such as [], then nothing will be yielded. """ dev_path = self.get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() if suffixes is None: suffixes = self.yield_suffixes(device, partition) else: partition_path = os.path.join(dev_path, DATADIR, partition) suffixes = ( (os.path.join(partition_path, suffix), suffix) for suffix in suffixes) for suffix_path, suffix in suffixes: for object_hash in self._listdir(suffix_path): object_path = os.path.join(suffix_path, object_hash) for name in hash_cleanup_listdir( object_path, self.reclaim_age): ts, ext = name.rsplit('.', 1) yield (object_path, object_hash, ts) break class DiskFileWriter(object): """ Encapsulation of the write context for servicing PUT REST API requests. Serves as the context manager object for the :class:`swift.obj.diskfile.DiskFile` class's :func:`swift.obj.diskfile.DiskFile.create` method. .. note:: It is the responsibility of the :func:`swift.obj.diskfile.DiskFile.create` method context manager to close the open file descriptor. .. note:: The arguments to the constructor are considered implementation specific. The API does not define the constructor arguments. :param name: name of object from REST API :param datadir: on-disk directory object will end up in on :func:`swift.obj.diskfile.DiskFileWriter.put` :param fd: open file descriptor of temporary file to receive data :param tmppath: full path name of the opened file descriptor :param bytes_per_sync: number bytes written between sync calls :param threadpool: internal thread pool to use for disk operations """ def __init__(self, name, datadir, fd, tmppath, bytes_per_sync, threadpool): # Parameter tracking self._name = name self._datadir = datadir self._fd = fd self._tmppath = tmppath self._bytes_per_sync = bytes_per_sync self._threadpool = threadpool # Internal attributes self._upload_size = 0 self._last_sync = 0 self._extension = '.data' def write(self, chunk): """ Write a chunk of data to disk. All invocations of this method must come before invoking the :func: For this implementation, the data is written into a temporary file. :param chunk: the chunk of data to write as a string object :returns: the total number of bytes written to an object """ def _write_entire_chunk(chunk): while chunk: written = os.write(self._fd, chunk) self._upload_size += written chunk = chunk[written:] self._threadpool.run_in_thread(_write_entire_chunk, chunk) # For large files sync every 512MB (by default) written diff = self._upload_size - self._last_sync if diff >= self._bytes_per_sync: self._threadpool.force_run_in_thread(fdatasync, self._fd) drop_buffer_cache(self._fd, self._last_sync, diff) self._last_sync = self._upload_size return self._upload_size def _finalize_put(self, metadata, target_path): # Write the metadata before calling fsync() so that both data and # metadata are flushed to disk. write_metadata(self._fd, metadata) # We call fsync() before calling drop_cache() to lower the amount of # redundant work the drop cache code will perform on the pages (now # that after fsync the pages will be all clean). fsync(self._fd) # From the Department of the Redundancy Department, make sure we call # drop_cache() after fsync() to avoid redundant work (pages all # clean). drop_buffer_cache(self._fd, 0, self._upload_size) invalidate_hash(dirname(self._datadir)) # After the rename completes, this object will be available for other # requests to reference. renamer(self._tmppath, target_path) try: hash_cleanup_listdir(self._datadir) except OSError: logging.exception(_('Problem cleaning up %s'), self._datadir) def put(self, metadata): """ Finalize writing the file on disk. For this implementation, this method is responsible for renaming the temporary file to the final name and directory location. This method should be called after the final call to :func:`swift.obj.diskfile.DiskFileWriter.write`. :param metadata: dictionary of metadata to be associated with the object """ timestamp = normalize_timestamp(metadata['X-Timestamp']) metadata['name'] = self._name target_path = join(self._datadir, timestamp + self._extension) self._threadpool.force_run_in_thread( self._finalize_put, metadata, target_path) class DiskFileReader(object): """ Encapsulation of the WSGI read context for servicing GET REST API requests. Serves as the context manager object for the :class:`swift.obj.diskfile.DiskFile` class's :func:`swift.obj.diskfile.DiskFile.reader` method. .. note:: The quarantining behavior of this method is considered implementation specific, and is not required of the API. .. note:: The arguments to the constructor are considered implementation specific. The API does not define the constructor arguments. :param fp: open file object pointer reference :param data_file: on-disk data file name for the object :param obj_size: verified on-disk size of the object :param etag: expected metadata etag value for entire file :param threadpool: thread pool to use for read operations :param disk_chunk_size: size of reads from disk in bytes :param keep_cache_size: maximum object size that will be kept in cache :param device_path: on-disk device path, used when quarantining an obj :param logger: logger caller wants this object to use :param quarantine_hook: 1-arg callable called w/reason when quarantined :param keep_cache: should resulting reads be kept in the buffer cache """ def __init__(self, fp, data_file, obj_size, etag, threadpool, disk_chunk_size, keep_cache_size, device_path, logger, quarantine_hook, keep_cache=False): # Parameter tracking self._fp = fp self._data_file = data_file self._obj_size = obj_size self._etag = etag self._threadpool = threadpool self._disk_chunk_size = disk_chunk_size self._device_path = device_path self._logger = logger self._quarantine_hook = quarantine_hook if keep_cache: # Caller suggests we keep this in cache, only do it if the # object's size is less than the maximum. self._keep_cache = obj_size < keep_cache_size else: self._keep_cache = False # Internal Attributes self._iter_etag = None self._bytes_read = 0 self._started_at_0 = False self._read_to_eof = False self._suppress_file_closing = False self._quarantined_dir = None def __iter__(self): """Returns an iterator over the data file.""" try: dropped_cache = 0 self._bytes_read = 0 self._started_at_0 = False self._read_to_eof = False if self._fp.tell() == 0: self._started_at_0 = True self._iter_etag = hashlib.md5() while True: chunk = self._threadpool.run_in_thread( self._fp.read, self._disk_chunk_size) if chunk: if self._iter_etag: self._iter_etag.update(chunk) self._bytes_read += len(chunk) if self._bytes_read - dropped_cache > (1024 * 1024): self._drop_cache(self._fp.fileno(), dropped_cache, self._bytes_read - dropped_cache) dropped_cache = self._bytes_read yield chunk else: self._read_to_eof = True self._drop_cache(self._fp.fileno(), dropped_cache, self._bytes_read - dropped_cache) break finally: if not self._suppress_file_closing: self.close() def app_iter_range(self, start, stop): """Returns an iterator over the data file for range (start, stop)""" if start or start == 0: self._fp.seek(start) if stop is not None: length = stop - start else: length = None try: for chunk in self: if length is not None: length -= len(chunk) if length < 0: # Chop off the extra: yield chunk[:length] break yield chunk finally: if not self._suppress_file_closing: self.close() def app_iter_ranges(self, ranges, content_type, boundary, size): """Returns an iterator over the data file for a set of ranges""" if not ranges: yield '' else: try: self._suppress_file_closing = True for chunk in multi_range_iterator( ranges, content_type, boundary, size, self.app_iter_range): yield chunk finally: self._suppress_file_closing = False self.close() def _drop_cache(self, fd, offset, length): """Method for no-oping buffer cache drop method.""" if not self._keep_cache: drop_buffer_cache(fd, offset, length) def _quarantine(self, msg): self._quarantined_dir = self._threadpool.run_in_thread( quarantine_renamer, self._device_path, self._data_file) self._logger.warn("Quarantined object %s: %s" % ( self._data_file, msg)) self._logger.increment('quarantines') self._quarantine_hook(msg) def _handle_close_quarantine(self): """Check if file needs to be quarantined""" if self._bytes_read != self._obj_size: self._quarantine( "Bytes read: %s, does not match metadata: %s" % ( self._bytes_read, self._obj_size)) elif self._iter_etag and \ self._etag != self._iter_etag.hexdigest(): self._quarantine( "ETag %s and file's md5 %s do not match" % ( self._etag, self._iter_etag.hexdigest())) def close(self): """ Close the open file handle if present. For this specific implementation, this method will handle quarantining the file if necessary. """ if self._fp: try: if self._started_at_0 and self._read_to_eof: self._handle_close_quarantine() except DiskFileQuarantined: raise except (Exception, Timeout) as e: self._logger.error(_( 'ERROR DiskFile %(data_file)s' ' close failure: %(exc)s : %(stack)s'), {'exc': e, 'stack': ''.join(traceback.format_stack()), 'data_file': self._data_file}) finally: fp, self._fp = self._fp, None fp.close() class DiskFile(object): """ Manage object files. This specific implementation manages object files on a disk formatted with a POSIX-compliant file system that supports extended attributes as metadata on a file or directory. .. note:: The arguments to the constructor are considered implementation specific. The API does not define the constructor arguments. :param mgr: associated DiskFileManager instance :param device_path: path to the target device or drive :param threadpool: thread pool to use for blocking operations :param partition: partition on the device in which the object lives :param account: account name for the object :param container: container name for the object :param obj: object name for the object """ def __init__(self, mgr, device_path, threadpool, partition, account=None, container=None, obj=None, _datadir=None): self._mgr = mgr self._device_path = device_path self._threadpool = threadpool or ThreadPool(nthreads=0) self._logger = mgr.logger self._disk_chunk_size = mgr.disk_chunk_size self._bytes_per_sync = mgr.bytes_per_sync if account and container and obj: self._name = '/' + '/'.join((account, container, obj)) self._account = account self._container = container self._obj = obj name_hash = hash_path(account, container, obj) self._datadir = join( device_path, storage_directory(DATADIR, partition, name_hash)) else: # gets populated when we read the metadata self._name = None self._account = None self._container = None self._obj = None self._datadir = None self._tmpdir = join(device_path, 'tmp') self._metadata = None self._data_file = None self._fp = None self._quarantined_dir = None self._content_length = None if _datadir: self._datadir = _datadir else: name_hash = hash_path(account, container, obj) self._datadir = join( device_path, storage_directory(DATADIR, partition, name_hash)) @property def account(self): return self._account @property def container(self): return self._container @property def obj(self): return self._obj @property def content_length(self): if self._metadata is None: raise DiskFileNotOpen() return self._content_length @property def timestamp(self): if self._metadata is None: raise DiskFileNotOpen() return self._metadata.get('X-Timestamp') @classmethod def from_hash_dir(cls, mgr, hash_dir_path, device_path, partition): return cls(mgr, device_path, None, partition, _datadir=hash_dir_path) def open(self): """ Open the object. This implementation opens the data file representing the object, reads the associated metadata in the extended attributes, additionally combining metadata from fast-POST `.meta` files. .. note:: An implementation is allowed to raise any of the following exceptions, but is only required to raise `DiskFileNotExist` when the object representation does not exist. :raises DiskFileCollision: on name mis-match with metadata :raises DiskFileNotExist: if the object does not exist :raises DiskFileDeleted: if the object was previously deleted :raises DiskFileQuarantined: if while reading metadata of the file some data did pass cross checks :returns: itself for use as a context manager """ data_file, meta_file, ts_file = self._get_ondisk_file() if not data_file: raise self._construct_exception_from_ts_file(ts_file) self._fp = self._construct_from_data_file( data_file, meta_file) # This method must populate the internal _metadata attribute. self._metadata = self._metadata or {} self._data_file = data_file return self def __enter__(self): """ Context enter. .. note:: An implemenation shall raise `DiskFileNotOpen` when has not previously invoked the :func:`swift.obj.diskfile.DiskFile.open` method. """ if self._metadata is None: raise DiskFileNotOpen() return self def __exit__(self, t, v, tb): """ Context exit. .. note:: This method will be invoked by the object server while servicing the REST API *before* the object has actually been read. It is the responsibility of the implementation to properly handle that. """ if self._fp is not None: fp, self._fp = self._fp, None fp.close() def _quarantine(self, data_file, msg): """ Quarantine a file; responsible for incrementing the associated logger's count of quarantines. :param data_file: full path of data file to quarantine :param msg: reason for quarantining to be included in the exception :returns: DiskFileQuarantined exception object """ self._quarantined_dir = self._threadpool.run_in_thread( quarantine_renamer, self._device_path, data_file) self._logger.warn("Quarantined object %s: %s" % ( data_file, msg)) self._logger.increment('quarantines') return DiskFileQuarantined(msg) def _get_ondisk_file(self): """ Do the work to figure out if the data directory exists, and if so, determine the on-disk files to use. :returns: a tuple of data, meta and ts (tombstone) files, in one of three states: * all three are None data directory does not exist, or there are no files in that directory * ts_file is not None, data_file is None, meta_file is None object is considered deleted * data_file is not None, ts_file is None object exists, and optionally has fast-POST metadata """ try: files = os.listdir(self._datadir) except OSError as err: if err.errno == errno.ENOTDIR: # If there's a file here instead of a directory, quarantine # it; something's gone wrong somewhere. raise self._quarantine( # hack: quarantine_renamer actually renames the directory # enclosing the filename you give it, but here we just # want this one file and not its parent. os.path.join(self._datadir, "made-up-filename"), "Expected directory, found file at %s" % self._datadir) elif err.errno != errno.ENOENT: raise DiskFileError( "Error listing directory %s: %s" % (self._datadir, err)) # The data directory does not exist, so the object cannot exist. fileset = (None, None, None) else: fileset = get_ondisk_files(files, self._datadir) return fileset def _construct_exception_from_ts_file(self, ts_file): """ If a tombstone is present it means the object is considered deleted. We just need to pull the metadata from the tombstone file which has the timestamp to construct the deleted exception. If there was no tombstone, just report it does not exist. :param ts_file: the tombstone file name found on disk :returns: DiskFileDeleted if the ts_file was provided, else DiskFileNotExist """ if not ts_file: exc = DiskFileNotExist() else: try: metadata = self._failsafe_read_metadata(ts_file, ts_file) except DiskFileQuarantined: # If the tombstone's corrupted, quarantine it and pretend it # wasn't there exc = DiskFileNotExist() else: # All well and good that we have found a tombstone file, but # we don't have a data file so we are just going to raise an # exception that we could not find the object, providing the # tombstone's timestamp. exc = DiskFileDeleted(metadata=metadata) return exc def _verify_name_matches_hash(self, data_file): hash_from_fs = os.path.basename(self._datadir) hash_from_name = hash_path(self._name.lstrip('/')) if hash_from_fs != hash_from_name: raise self._quarantine( data_file, "Hash of name in metadata does not match directory name") def _verify_data_file(self, data_file, fp): """ Verify the metadata's name value matches what we think the object is named. :param data_file: data file name being consider, used when quarantines occur :param fp: open file pointer so that we can `fstat()` the file to verify the on-disk size with Content-Length metadata value :raises DiskFileCollision: if the metadata stored name does not match the referenced name of the file :raises DiskFileExpired: if the object has expired :raises DiskFileQuarantined: if data inconsistencies were detected between the metadata and the file-system metadata """ try: mname = self._metadata['name'] except KeyError: raise self._quarantine(data_file, "missing name metadata") else: if mname != self._name: self._logger.error( _('Client path %(client)s does not match ' 'path stored in object metadata %(meta)s'), {'client': self._name, 'meta': mname}) raise DiskFileCollision('Client path does not match path ' 'stored in object metadata') try: x_delete_at = int(self._metadata['X-Delete-At']) except KeyError: pass except ValueError: # Quarantine, the x-delete-at key is present but not an # integer. raise self._quarantine( data_file, "bad metadata x-delete-at value %s" % ( self._metadata['X-Delete-At'])) else: if x_delete_at <= time.time(): raise DiskFileExpired(metadata=self._metadata) try: metadata_size = int(self._metadata['Content-Length']) except KeyError: raise self._quarantine( data_file, "missing content-length in metadata") except ValueError: # Quarantine, the content-length key is present but not an # integer. raise self._quarantine( data_file, "bad metadata content-length value %s" % ( self._metadata['Content-Length'])) fd = fp.fileno() try: statbuf = os.fstat(fd) except OSError as err: # Quarantine, we can't successfully stat the file. raise self._quarantine(data_file, "not stat-able: %s" % err) else: obj_size = statbuf.st_size if obj_size != metadata_size: raise self._quarantine( data_file, "metadata content-length %s does" " not match actual object size %s" % ( metadata_size, statbuf.st_size)) self._content_length = obj_size return obj_size def _failsafe_read_metadata(self, source, quarantine_filename=None): # Takes source and filename separately so we can read from an open # file if we have one try: return read_metadata(source) except Exception as err: raise self._quarantine( quarantine_filename, "Exception reading metadata: %s" % err) def _construct_from_data_file(self, data_file, meta_file): """ Open the `.data` file to fetch its metadata, and fetch the metadata from the fast-POST `.meta` file as well if it exists, merging them properly. :param data_file: on-disk `.data` file being considered :param meta_file: on-disk fast-POST `.meta` file being considered :returns: an opened data file pointer :raises DiskFileError: various exceptions from :func:`swift.obj.diskfile.DiskFile._verify_data_file` """ fp = open(data_file, 'rb') datafile_metadata = self._failsafe_read_metadata(fp, data_file) if meta_file: self._metadata = self._failsafe_read_metadata(meta_file, meta_file) sys_metadata = dict( [(key, val) for key, val in datafile_metadata.iteritems() if key.lower() in DATAFILE_SYSTEM_META]) self._metadata.update(sys_metadata) else: self._metadata = datafile_metadata if self._name is None: # If we don't know our name, we were just given a hash dir at # instantiation, so we'd better validate that the name hashes back # to us self._name = self._metadata['name'] self._verify_name_matches_hash(data_file) self._verify_data_file(data_file, fp) return fp def get_metadata(self): """ Provide the metadata for a previously opened object as a dictionary. :returns: object's metadata dictionary :raises DiskFileNotOpen: if the :func:`swift.obj.diskfile.DiskFile.open` method was not previously invoked """ if self._metadata is None: raise DiskFileNotOpen() return self._metadata def read_metadata(self): """ Return the metadata for an object without requiring the caller to open the object first. :returns: metadata dictionary for an object :raises DiskFileError: this implementation will raise the same errors as the `open()` method. """ with self.open(): return self.get_metadata() def reader(self, keep_cache=False, _quarantine_hook=lambda m: None): """ Return a :class:`swift.common.swob.Response` class compatible "`app_iter`" object as defined by :class:`swift.obj.diskfile.DiskFileReader`. For this implementation, the responsibility of closing the open file is passed to the :class:`swift.obj.diskfile.DiskFileReader` object. :param keep_cache: caller's preference for keeping data read in the OS buffer cache :param _quarantine_hook: 1-arg callable called when obj quarantined; the arg is the reason for quarantine. Default is to ignore it. Not needed by the REST layer. :returns: a :class:`swift.obj.diskfile.DiskFileReader` object """ dr = DiskFileReader( self._fp, self._data_file, int(self._metadata['Content-Length']), self._metadata['ETag'], self._threadpool, self._disk_chunk_size, self._mgr.keep_cache_size, self._device_path, self._logger, quarantine_hook=_quarantine_hook, keep_cache=keep_cache) # At this point the reader object is now responsible for closing # the file pointer. self._fp = None return dr @contextmanager def create(self, size=None): """ Context manager to create a file. We create a temporary file first, and then return a DiskFileWriter object to encapsulate the state. .. note:: An implementation is not required to perform on-disk preallocations even if the parameter is specified. But if it does and it fails, it must raise a `DiskFileNoSpace` exception. :param size: optional initial size of file to explicitly allocate on disk :raises DiskFileNoSpace: if a size is specified and allocation fails """ if not exists(self._tmpdir): mkdirs(self._tmpdir) fd, tmppath = mkstemp(dir=self._tmpdir) try: if size is not None and size > 0: try: fallocate(fd, size) except OSError: raise DiskFileNoSpace() yield DiskFileWriter(self._name, self._datadir, fd, tmppath, self._bytes_per_sync, self._threadpool) finally: try: os.close(fd) except OSError: pass try: os.unlink(tmppath) except OSError: pass def write_metadata(self, metadata): """ Write a block of metadata to an object without requiring the caller to create the object first. Supports fast-POST behavior semantics. :param metadata: dictionary of metadata to be associated with the object :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ with self.create() as writer: writer._extension = '.meta' writer.put(metadata) def delete(self, timestamp): """ Delete the object. This implementation creates a tombstone file using the given timestamp, and removes any older versions of the object file. Any file that has an older timestamp than timestamp will be deleted. .. note:: An implementation is free to use or ignore the timestamp parameter. :param timestamp: timestamp to compare with each file :raises DiskFileError: this implementation will raise the same errors as the `create()` method. """ timestamp = normalize_timestamp(timestamp) with self.create() as deleter: deleter._extension = '.ts' deleter.put({'X-Timestamp': timestamp}) swift-1.13.1/swift/obj/__init__.py0000664000175400017540000000000012323703611020072 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/cli/0000775000175400017540000000000012323703665016001 5ustar jenkinsjenkins00000000000000swift-1.13.1/swift/cli/ringbuilder.py0000775000175400017540000010351112323703611020654 0ustar jenkinsjenkins00000000000000#! /usr/bin/env python # Copyright (c) 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from array import array from errno import EEXIST from itertools import islice, izip from math import ceil from os import mkdir from os.path import basename, abspath, dirname, exists, join as pathjoin from sys import argv as sys_argv, exit, stderr from textwrap import wrap from time import time from swift.common import exceptions from swift.common.ring import RingBuilder, Ring from swift.common.ring.builder import MAX_BALANCE from swift.common.utils import lock_parent_directory from swift.common.ring.utils import parse_search_value, parse_args, \ build_dev_from_opts, parse_builder_ring_filename_args MAJOR_VERSION = 1 MINOR_VERSION = 3 EXIT_SUCCESS = 0 EXIT_WARNING = 1 EXIT_ERROR = 2 global argv, backup_dir, builder, builder_file, ring_file argv = backup_dir = builder = builder_file = ring_file = None def format_device(dev): """ Format a device for display. """ copy_dev = dev.copy() for key in ('ip', 'replication_ip'): if ':' in copy_dev[key]: copy_dev[key] = '[' + copy_dev[key] + ']' return ('d%(id)sr%(region)sz%(zone)s-%(ip)s:%(port)sR' '%(replication_ip)s:%(replication_port)s/%(device)s_' '"%(meta)s"' % copy_dev) def _parse_add_values(argvish): """ Parse devices to add as specified on the command line. Will exit on error and spew warnings. :returns: array of device dicts """ opts, args = parse_args(argvish) # We'll either parse the all-in-one-string format or the --options format, # but not both. If both are specified, raise an error. opts_used = opts.region or opts.zone or opts.ip or opts.port or \ opts.device or opts.weight or opts.meta if len(args) > 0 and opts_used: print Commands.add.__doc__.strip() exit(EXIT_ERROR) elif len(args) > 0: if len(args) % 2 != 0: print Commands.add.__doc__.strip() exit(EXIT_ERROR) parsed_devs = [] devs_and_weights = izip(islice(args, 0, len(args), 2), islice(args, 1, len(args), 2)) for devstr, weightstr in devs_and_weights: region = 1 rest = devstr if devstr.startswith('r'): i = 1 while i < len(devstr) and devstr[i].isdigit(): i += 1 region = int(devstr[1:i]) rest = devstr[i:] else: stderr.write("WARNING: No region specified for %s. " "Defaulting to region 1.\n" % devstr) if not rest.startswith('z'): print 'Invalid add value: %s' % devstr exit(EXIT_ERROR) i = 1 while i < len(rest) and rest[i].isdigit(): i += 1 zone = int(rest[1:i]) rest = rest[i:] if not rest.startswith('-'): print 'Invalid add value: %s' % devstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) i = 1 if rest[i] == '[': i += 1 while i < len(rest) and rest[i] != ']': i += 1 i += 1 ip = rest[1:i].lstrip('[').rstrip(']') rest = rest[i:] else: while i < len(rest) and rest[i] in '0123456789.': i += 1 ip = rest[1:i] rest = rest[i:] if not rest.startswith(':'): print 'Invalid add value: %s' % devstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) i = 1 while i < len(rest) and rest[i].isdigit(): i += 1 port = int(rest[1:i]) rest = rest[i:] replication_ip = ip replication_port = port if rest.startswith('R'): i = 1 if rest[i] == '[': i += 1 while i < len(rest) and rest[i] != ']': i += 1 i += 1 replication_ip = rest[1:i].lstrip('[').rstrip(']') rest = rest[i:] else: while i < len(rest) and rest[i] in '0123456789.': i += 1 replication_ip = rest[1:i] rest = rest[i:] if not rest.startswith(':'): print 'Invalid add value: %s' % devstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) i = 1 while i < len(rest) and rest[i].isdigit(): i += 1 replication_port = int(rest[1:i]) rest = rest[i:] if not rest.startswith('/'): print 'Invalid add value: %s' % devstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) i = 1 while i < len(rest) and rest[i] != '_': i += 1 device_name = rest[1:i] rest = rest[i:] meta = '' if rest.startswith('_'): meta = rest[1:] try: weight = float(weightstr) except ValueError: print 'Invalid weight value: %s' % weightstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) if weight < 0: print 'Invalid weight value (must be positive): %s' % weightstr print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) parsed_devs.append({'region': region, 'zone': zone, 'ip': ip, 'port': port, 'device': device_name, 'replication_ip': replication_ip, 'replication_port': replication_port, 'weight': weight, 'meta': meta}) return parsed_devs else: try: dev = build_dev_from_opts(opts) except ValueError as e: print e print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) return [dev] class Commands(object): def unknown(): print 'Unknown command: %s' % argv[2] exit(EXIT_ERROR) def create(): """ swift-ring-builder create Creates with 2^ partitions and . is number of hours to restrict moving a partition more than once. """ if len(argv) < 6: print Commands.create.__doc__.strip() exit(EXIT_ERROR) builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5])) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1]))) builder.save(argv[1]) exit(EXIT_SUCCESS) def default(): """ swift-ring-builder Shows information about the ring and the devices within. """ print '%s, build version %d' % (argv[1], builder.version) regions = 0 zones = 0 balance = 0 dev_count = 0 if builder.devs: regions = len(set(d['region'] for d in builder.devs if d is not None)) zones = len(set((d['region'], d['zone']) for d in builder.devs if d is not None)) dev_count = len([dev for dev in builder.devs if dev is not None]) balance = builder.get_balance() print '%d partitions, %.6f replicas, %d regions, %d zones, ' \ '%d devices, %.02f balance' % (builder.parts, builder.replicas, regions, zones, dev_count, balance) print 'The minimum number of hours before a partition can be ' \ 'reassigned is %s' % builder.min_part_hours if builder.devs: print 'Devices: id region zone ip address port ' \ 'replication ip replication port name ' \ 'weight partitions balance meta' weighted_parts = builder.parts * builder.replicas / \ sum(d['weight'] for d in builder.devs if d is not None) for dev in builder.devs: if dev is None: continue if not dev['weight']: if dev['parts']: balance = MAX_BALANCE else: balance = 0 else: balance = 100.0 * dev['parts'] / \ (dev['weight'] * weighted_parts) - 100.0 print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f ' '%10s %7.02f %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance, dev['meta'])) exit(EXIT_SUCCESS) def search(): """ swift-ring-builder search Shows information about matching devices. """ if len(argv) < 4: print Commands.search.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) devs = builder.search_devs(parse_search_value(argv[3])) if not devs: print 'No matching devices found' exit(EXIT_ERROR) print 'Devices: id region zone ip address port ' \ 'replication ip replication port name weight partitions ' \ 'balance meta' weighted_parts = builder.parts * builder.replicas / \ sum(d['weight'] for d in builder.devs if d is not None) for dev in devs: if not dev['weight']: if dev['parts']: balance = MAX_BALANCE else: balance = 0 else: balance = 100.0 * dev['parts'] / \ (dev['weight'] * weighted_parts) - 100.0 print(' %5d %7d %5d %15s %5d %15s %17d %9s %6.02f %10s ' '%7.02f %s' % (dev['id'], dev['region'], dev['zone'], dev['ip'], dev['port'], dev['replication_ip'], dev['replication_port'], dev['device'], dev['weight'], dev['parts'], balance, dev['meta'])) exit(EXIT_SUCCESS) def list_parts(): """ swift-ring-builder list_parts [] .. Returns a 2 column list of all the partitions that are assigned to any of the devices matching the search values given. The first column is the assigned partition number and the second column is the number of device matches for that partition. The list is ordered from most number of matches to least. If there are a lot of devices to match against, this command could take a while to run. """ if len(argv) < 4: print Commands.list_parts.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) devs = [] for arg in argv[3:]: devs.extend(builder.search_devs(parse_search_value(arg)) or []) if not devs: print 'No matching devices found' exit(EXIT_ERROR) devs = [d['id'] for d in devs] max_replicas = int(ceil(builder.replicas)) matches = [array('i') for x in xrange(max_replicas)] for part in xrange(builder.parts): count = len([d for d in builder.get_part_devices(part) if d['id'] in devs]) if count: matches[max_replicas - count].append(part) print 'Partition Matches' for index, parts in enumerate(matches): for part in parts: print '%9d %7d' % (part, max_replicas - index) exit(EXIT_SUCCESS) def add(): """ swift-ring-builder add [r]z-:[R:]/_ [[r]z-:[R:]/_ ] ... Where and are replication ip and port. or swift-ring-builder add [--region ] --zone --ip --port --replication-ip --replication-port --device --meta --weight Adds devices to the ring with the given information. No partitions will be assigned to the new device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. """ if len(argv) < 5 or len(argv) % 2 != 1: print Commands.add.__doc__.strip() exit(EXIT_ERROR) for new_dev in _parse_add_values(argv[3:]): for dev in builder.devs: if dev is None: continue if dev['ip'] == new_dev['ip'] and \ dev['port'] == new_dev['port'] and \ dev['device'] == new_dev['device']: print 'Device %d already uses %s:%d/%s.' % \ (dev['id'], dev['ip'], dev['port'], dev['device']) print "The on-disk ring builder is unchanged.\n" exit(EXIT_ERROR) dev_id = builder.add_dev(new_dev) print('Device %s with %s weight got id %s' % (format_device(new_dev), new_dev['weight'], dev_id)) builder.save(argv[1]) exit(EXIT_SUCCESS) def set_weight(): """ swift-ring-builder set_weight [ 1: print 'Matched more than one device:' for dev in devs: print ' %s' % format_device(dev) if raw_input('Are you sure you want to update the weight for ' 'these %s devices? (y/N) ' % len(devs)) != 'y': print 'Aborting device modifications' exit(EXIT_ERROR) for dev in devs: builder.set_dev_weight(dev['id'], weight) print '%s weight set to %s' % (format_device(dev), dev['weight']) builder.save(argv[1]) exit(EXIT_SUCCESS) def set_info(): """ swift-ring-builder set_info :[R:]/_ [ :[R:]/_] ... Where and are replication ip and port. For each search-value, resets the matched device's information. This information isn't used to assign partitions, so you can use 'write_ring' afterward to rewrite the current ring with the newer device information. Any of the parts are optional in the final :/_ parameter; just give what you want to change. For instance set_info d74 _"snet: 5.6.7.8" would just update the meta data for device id 74. """ if len(argv) < 5 or len(argv) % 2 != 1: print Commands.set_info.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) searches_and_changes = izip(islice(argv, 3, len(argv), 2), islice(argv, 4, len(argv), 2)) for search_value, change_value in searches_and_changes: devs = builder.search_devs(parse_search_value(search_value)) change = [] if len(change_value) and change_value[0].isdigit(): i = 1 while (i < len(change_value) and change_value[i] in '0123456789.'): i += 1 change.append(('ip', change_value[:i])) change_value = change_value[i:] elif len(change_value) and change_value[0] == '[': i = 1 while i < len(change_value) and change_value[i] != ']': i += 1 i += 1 change.append(('ip', change_value[:i].lstrip('[').rstrip(']'))) change_value = change_value[i:] if change_value.startswith(':'): i = 1 while i < len(change_value) and change_value[i].isdigit(): i += 1 change.append(('port', int(change_value[1:i]))) change_value = change_value[i:] if change_value.startswith('R'): change_value = change_value[1:] if len(change_value) and change_value[0].isdigit(): i = 1 while (i < len(change_value) and change_value[i] in '0123456789.'): i += 1 change.append(('replication_ip', change_value[:i])) change_value = change_value[i:] elif len(change_value) and change_value[0] == '[': i = 1 while i < len(change_value) and change_value[i] != ']': i += 1 i += 1 change.append(('replication_ip', change_value[:i].lstrip('[').rstrip(']'))) change_value = change_value[i:] if change_value.startswith(':'): i = 1 while i < len(change_value) and change_value[i].isdigit(): i += 1 change.append(('replication_port', int(change_value[1:i]))) change_value = change_value[i:] if change_value.startswith('/'): i = 1 while i < len(change_value) and change_value[i] != '_': i += 1 change.append(('device', change_value[1:i])) change_value = change_value[i:] if change_value.startswith('_'): change.append(('meta', change_value[1:])) change_value = '' if change_value or not change: raise ValueError('Invalid set info change value: %s' % repr(argv[4])) if not devs: print("Search value \"%s\" matched 0 devices.\n" "The on-disk ring builder is unchanged.\n" % search_value) exit(EXIT_ERROR) if len(devs) > 1: print 'Matched more than one device:' for dev in devs: print ' %s' % format_device(dev) if raw_input('Are you sure you want to update the info for ' 'these %s devices? (y/N) ' % len(devs)) != 'y': print 'Aborting device modifications' exit(EXIT_ERROR) for dev in devs: orig_dev_string = format_device(dev) test_dev = dict(dev) for key, value in change: test_dev[key] = value for check_dev in builder.devs: if not check_dev or check_dev['id'] == test_dev['id']: continue if check_dev['ip'] == test_dev['ip'] and \ check_dev['port'] == test_dev['port'] and \ check_dev['device'] == test_dev['device']: print 'Device %d already uses %s:%d/%s.' % \ (check_dev['id'], check_dev['ip'], check_dev['port'], check_dev['device']) exit(EXIT_ERROR) for key, value in change: dev[key] = value print 'Device %s is now %s' % (orig_dev_string, format_device(dev)) builder.save(argv[1]) exit(EXIT_SUCCESS) def remove(): """ swift-ring-builder remove [search-value ...] Removes the device(s) from the ring. This should normally just be used for a device that has failed. For a device you wish to decommission, it's best to set its weight to 0, wait for it to drain all its data, then use this remove command. This will not take effect until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. """ if len(argv) < 4: print Commands.remove.__doc__.strip() print print parse_search_value.__doc__.strip() exit(EXIT_ERROR) for search_value in argv[3:]: devs = builder.search_devs(parse_search_value(search_value)) if not devs: print("Search value \"%s\" matched 0 devices.\n" "The on-disk ring builder is unchanged." % search_value) exit(EXIT_ERROR) if len(devs) > 1: print 'Matched more than one device:' for dev in devs: print ' %s' % format_device(dev) if raw_input('Are you sure you want to remove these %s ' 'devices? (y/N) ' % len(devs)) != 'y': print 'Aborting device removals' exit(EXIT_ERROR) for dev in devs: try: builder.remove_dev(dev['id']) except exceptions.RingBuilderError as e: print '-' * 79 print( "An error occurred while removing device with id %d\n" "This usually means that you attempted to remove\n" "the last device in a ring. If this is the case,\n" "consider creating a new ring instead.\n" "The on-disk ring builder is unchanged.\n" "Original exception message: %s" % (dev['id'], e.message) ) print '-' * 79 exit(EXIT_ERROR) print '%s marked for removal and will ' \ 'be removed next rebalance.' % format_device(dev) builder.save(argv[1]) exit(EXIT_SUCCESS) def rebalance(): """ swift-ring-builder rebalance Attempts to rebalance the ring by reassigning partitions that haven't been recently reassigned. """ def get_seed(index): try: return argv[index] except IndexError: pass devs_changed = builder.devs_changed try: last_balance = builder.get_balance() parts, balance = builder.rebalance(seed=get_seed(3)) except exceptions.RingBuilderError as e: print '-' * 79 print("An error has occurred during ring validation. Common\n" "causes of failure are rings that are empty or do not\n" "have enough devices to accommodate the replica count.\n" "Original exception message:\n %s" % e.message ) print '-' * 79 exit(EXIT_ERROR) if not parts: print 'No partitions could be reassigned.' print 'Either none need to be or none can be due to ' \ 'min_part_hours [%s].' % builder.min_part_hours exit(EXIT_WARNING) # If we set device's weight to zero, currently balance will be set # special value(MAX_BALANCE) until zero weighted device return all # its partitions. So we cannot check balance has changed. # Thus we need to check balance or last_balance is special value. if not devs_changed and abs(last_balance - balance) < 1 and \ not (last_balance == MAX_BALANCE and balance == MAX_BALANCE): print 'Cowardly refusing to save rebalance as it did not change ' \ 'at least 1%.' exit(EXIT_WARNING) try: builder.validate() except exceptions.RingValidationError as e: print '-' * 79 print("An error has occurred during ring validation. Common\n" "causes of failure are rings that are empty or do not\n" "have enough devices to accommodate the replica count.\n" "Original exception message:\n %s" % e.message ) print '-' * 79 exit(EXIT_ERROR) print 'Reassigned %d (%.02f%%) partitions. Balance is now %.02f.' % \ (parts, 100.0 * parts / builder.parts, balance) status = EXIT_SUCCESS if balance > 5: print '-' * 79 print 'NOTE: Balance of %.02f indicates you should push this ' % \ balance print ' ring, wait at least %d hours, and rebalance/repush.' \ % builder.min_part_hours print '-' * 79 status = EXIT_WARNING ts = time() builder.get_ring().save( pathjoin(backup_dir, '%d.' % ts + basename(ring_file))) builder.save(pathjoin(backup_dir, '%d.' % ts + basename(argv[1]))) builder.get_ring().save(ring_file) builder.save(argv[1]) exit(status) def validate(): """ swift-ring-builder validate Just runs the validation routines on the ring. """ builder.validate() exit(EXIT_SUCCESS) def write_ring(): """ swift-ring-builder write_ring Just rewrites the distributable ring file. This is done automatically after a successful rebalance, so really this is only useful after one or more 'set_info' calls when no rebalance is needed but you want to send out the new device information. """ ring_data = builder.get_ring() if not ring_data._replica2part2dev_id: if ring_data.devs: print 'Warning: Writing a ring with no partition ' \ 'assignments but with devices; did you forget to run ' \ '"rebalance"?' else: print 'Warning: Writing an empty ring' ring_data.save( pathjoin(backup_dir, '%d.' % time() + basename(ring_file))) ring_data.save(ring_file) exit(EXIT_SUCCESS) def write_builder(): """ swift-ring-builder write_builder [min_part_hours] Recreate a builder from a ring file (lossy) if you lost your builder backups. (Protip: don't lose your builder backups). [min_part_hours] is one of those numbers lost to the builder, you can change it with set_min_part_hours. """ if exists(builder_file): print 'Cowardly refusing to overwrite existing ' \ 'Ring Builder file: %s' % builder_file exit(EXIT_ERROR) if len(argv) > 3: min_part_hours = int(argv[3]) else: stderr.write("WARNING: default min_part_hours may not match " "the value in the lost builder.\n") min_part_hours = 24 ring = Ring(ring_file) for dev in ring.devs: dev.update({ 'parts': 0, 'parts_wanted': 0, }) builder_dict = { 'part_power': 32 - ring._part_shift, 'replicas': float(ring.replica_count), 'min_part_hours': min_part_hours, 'parts': ring.partition_count, 'devs': ring.devs, 'devs_changed': False, 'version': 0, '_replica2part2dev': ring._replica2part2dev_id, '_last_part_moves_epoch': None, '_last_part_moves': None, '_last_part_gather_start': 0, '_remove_devs': [], } builder = RingBuilder(1, 1, 1) builder.copy_from(builder_dict) for parts in builder._replica2part2dev: for dev_id in parts: builder.devs[dev_id]['parts'] += 1 builder._set_parts_wanted() builder.save(builder_file) def pretend_min_part_hours_passed(): builder.pretend_min_part_hours_passed() builder.save(argv[1]) exit(EXIT_SUCCESS) def set_min_part_hours(): """ swift-ring-builder set_min_part_hours Changes the to the given . This should be set to however long a full replication/update cycle takes. We're working on a way to determine this more easily than scanning logs. """ if len(argv) < 4: print Commands.set_min_part_hours.__doc__.strip() exit(EXIT_ERROR) builder.change_min_part_hours(int(argv[3])) print 'The minimum number of hours before a partition can be ' \ 'reassigned is now set to %s' % argv[3] builder.save(argv[1]) exit(EXIT_SUCCESS) def set_replicas(): """ swift-ring-builder set_replicas Changes the replica count to the given . may be a floating-point value, in which case some partitions will have floor() replicas and some will have ceiling() in the correct proportions. A rebalance is needed to make the change take effect. """ if len(argv) < 4: print Commands.set_replicas.__doc__.strip() exit(EXIT_ERROR) new_replicas = argv[3] try: new_replicas = float(new_replicas) except ValueError: print Commands.set_replicas.__doc__.strip() print "\"%s\" is not a valid number." % new_replicas exit(EXIT_ERROR) if new_replicas < 1: print "Replica count must be at least 1." exit(EXIT_ERROR) builder.set_replicas(new_replicas) print 'The replica count is now %.6f.' % builder.replicas print 'The change will take effect after the next rebalance.' builder.save(argv[1]) exit(EXIT_SUCCESS) def main(arguments=None): global argv, backup_dir, builder, builder_file, ring_file if arguments: argv = arguments else: argv = sys_argv if len(argv) < 2: print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \ globals() print Commands.default.__doc__.strip() print cmds = [c for c, f in Commands.__dict__.iteritems() if f.__doc__ and c[0] != '_' and c != 'default'] cmds.sort() for cmd in cmds: print Commands.__dict__[cmd].__doc__.strip() print print parse_search_value.__doc__.strip() print for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ', subsequent_indent=' '): print line print('Exit codes: 0 = operation successful\n' ' 1 = operation completed with warnings\n' ' 2 = error') exit(EXIT_SUCCESS) builder_file, ring_file = parse_builder_ring_filename_args(argv) if exists(builder_file): builder = RingBuilder.load(builder_file) elif len(argv) < 3 or argv[2] not in('create', 'write_builder'): print 'Ring Builder file does not exist: %s' % argv[1] exit(EXIT_ERROR) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise if len(argv) == 2: command = "default" else: command = argv[2] if argv[0].endswith('-safe'): try: with lock_parent_directory(abspath(argv[1]), 15): Commands.__dict__.get(command, Commands.unknown.im_func)() except exceptions.LockTimeout: print "Ring/builder dir currently locked." exit(2) else: Commands.__dict__.get(command, Commands.unknown.im_func)() if __name__ == '__main__': main() swift-1.13.1/swift/cli/info.py0000664000175400017540000001467312323703614017313 0ustar jenkinsjenkins00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy # of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os from datetime import datetime from swift.common.utils import hash_path, storage_directory from swift.common.ring import Ring from swift.common.request_helpers import is_sys_meta, is_user_meta, \ strip_sys_meta_prefix, strip_user_meta_prefix from swift.account.backend import AccountBroker, DATADIR as ABDATADIR from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR class InfoSystemExit(Exception): """ Indicates to the caller that a sys.exit(1) should be performed. """ pass def print_ring_locations(ring, datadir, account, container=None): """ print out ring locations of specified type :param ring: ring instance :param datadir: high level directory to store account/container/objects :param acount: account name :param container: container name """ if ring is None or datadir is None or account is None: raise ValueError('None type') storage_type = 'account' if container: storage_type = 'container' try: part, nodes = ring.get_nodes(account, container, None) except (ValueError, AttributeError): raise ValueError('Ring error') else: path_hash = hash_path(account, container, None) print '\nRing locations:' for node in nodes: print (' %s:%s - /srv/node/%s/%s/%s.db' % (node['ip'], node['port'], node['device'], storage_directory(datadir, part, path_hash), path_hash)) print '\nnote: /srv/node is used as default value of `devices`, the ' \ 'real value is set in the %s config file on each storage node.' % \ storage_type def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print 'Path: %s' % path print ' Account: %s' % account if db_type == 'container': print ' Container: %s' % container path_hash = hash_path(account, container) if db_type == 'container': print ' Container Hash: %s' % path_hash else: print ' Account Hash: %s' % path_hash print 'Metadata:' print (' Created at: %s (%s)' % (datetime.utcfromtimestamp(float(info['created_at'])), info['created_at'])) print (' Put Timestamp: %s (%s)' % (datetime.utcfromtimestamp(float(info['put_timestamp'])), info['put_timestamp'])) print (' Delete Timestamp: %s (%s)' % (datetime.utcfromtimestamp(float(info['delete_timestamp'])), info['delete_timestamp'])) print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': print (' Reported Put Timestamp: %s (%s)' % (datetime.utcfromtimestamp( float(info['reported_put_timestamp'])), info['reported_put_timestamp'])) print (' Reported Delete Timestamp: %s (%s)' % (datetime.utcfromtimestamp (float(info['reported_delete_timestamp'])), info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] print ' Chexor: %s' % info['hash'] print ' UUID: %s' % info['id'] except KeyError: raise ValueError('Info is incomplete') meta_prefix = 'x_' + db_type + '_' for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print ' %s: %s' % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace('_', '-').title() print ' %s: %s' % (title, value) if sys_metadata: print ' System Metadata: %s' % sys_metadata else: print 'No system metadata found in db file' if user_metadata: print ' User Metadata: %s' % user_metadata else: print 'No user metadata found in db file' def print_info(db_type, db_file, swift_dir='/etc/swift'): if db_type not in ('account', 'container'): print "Unrecognized DB type: internal error" raise InfoSystemExit() if not os.path.exists(db_file) or not db_file.endswith('.db'): print "DB file doesn't exist" raise InfoSystemExit() if not db_file.startswith(('/', './')): db_file = './' + db_file # don't break if the bare db file is given if db_type == 'account': broker = AccountBroker(db_file) datadir = ABDATADIR else: broker = ContainerBroker(db_file) datadir = CBDATADIR info = broker.get_info() account = info['account'] container = info['container'] if db_type == 'container' else None print_db_info_metadata(db_type, info, broker.metadata) try: ring = Ring(swift_dir, ring_name=db_type) except Exception: ring = None else: print_ring_locations(ring, datadir, account, container) swift-1.13.1/swift/cli/recon.py0000775000175400017540000011177112323703611017463 0ustar jenkinsjenkins00000000000000#! /usr/bin/env python # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. """ cmdline utility to perform cluster reconnaissance """ from eventlet.green import urllib2 from swift.common.ring import Ring from urlparse import urlparse try: import simplejson as json except ImportError: import json from hashlib import md5 import eventlet import optparse import time import sys import os def seconds2timeunit(seconds): elapsed = seconds unit = 'seconds' if elapsed >= 60: elapsed = elapsed / 60.0 unit = 'minutes' if elapsed >= 60: elapsed = elapsed / 60.0 unit = 'hours' if elapsed >= 24: elapsed = elapsed / 24.0 unit = 'days' return elapsed, unit def size_suffix(size): suffixes = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] for suffix in suffixes: if size < 1000: return "%s %s" % (size, suffix) size = size / 1000 return "%s %s" % (size, suffix) class Scout(object): """ Obtain swift recon information """ def __init__(self, recon_type, verbose=False, suppress_errors=False, timeout=5): self.recon_type = recon_type self.verbose = verbose self.suppress_errors = suppress_errors self.timeout = timeout def scout_host(self, base_url, recon_type): """ Perform the actual HTTP request to obtain swift recon telemtry. :param base_url: the base url of the host you wish to check. str of the format 'http://127.0.0.1:6000/recon/' :param recon_type: the swift recon check to request. :returns: tuple of (recon url used, response body, and status) """ url = base_url + recon_type try: body = urllib2.urlopen(url, timeout=self.timeout).read() content = json.loads(body) if self.verbose: print "-> %s: %s" % (url, content) status = 200 except urllib2.HTTPError as err: if not self.suppress_errors or self.verbose: print "-> %s: %s" % (url, err) content = err status = err.code except urllib2.URLError as err: if not self.suppress_errors or self.verbose: print "-> %s: %s" % (url, err) content = err status = -1 return url, content, status def scout(self, host): """ Obtain telemetry from a host running the swift recon middleware. :param host: host to check :returns: tuple of (recon url used, response body, and status) """ base_url = "http://%s:%s/recon/" % (host[0], host[1]) url, content, status = self.scout_host(base_url, self.recon_type) return url, content, status class SwiftRecon(object): """ Retrieve and report cluster info from hosts running recon middleware. """ def __init__(self): self.verbose = False self.suppress_errors = False self.timeout = 5 self.pool_size = 30 self.pool = eventlet.GreenPool(self.pool_size) self.check_types = ['account', 'container', 'object'] self.server_type = 'object' def _gen_stats(self, stats, name=None): """Compute various stats from a list of values.""" cstats = [x for x in stats if x is not None] if len(cstats) > 0: ret_dict = {'low': min(cstats), 'high': max(cstats), 'total': sum(cstats), 'reported': len(cstats), 'number_none': len(stats) - len(cstats), 'name': name} ret_dict['average'] = \ ret_dict['total'] / float(len(cstats)) ret_dict['perc_none'] = \ ret_dict['number_none'] * 100.0 / len(stats) else: ret_dict = {'reported': 0} return ret_dict def _print_stats(self, stats): """ print out formatted stats to console :param stats: dict of stats generated by _gen_stats """ print '[%(name)s] low: %(low)d, high: %(high)d, avg: ' \ '%(average).1f, total: %(total)d, ' \ 'Failed: %(perc_none).1f%%, no_result: %(number_none)d, ' \ 'reported: %(reported)d' % stats def _ptime(self, timev=None): """ :param timev: a unix timestamp or None :returns: a pretty string of the current time or provided time """ if timev: return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timev)) else: return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) def get_devices(self, zone_filter, swift_dir, ring_name): """ Get a list of hosts in the ring :param zone_filter: Only list zones matching given filter :param swift_dir: Directory of swift config, usually /etc/swift :param ring_name: Name of the ring, such as 'object' :returns: a set of tuples containing the ip and port of hosts """ ring_data = Ring(swift_dir, ring_name=ring_name) if zone_filter is not None: ips = set((n['ip'], n['port']) for n in ring_data.devs if n and n['zone'] == zone_filter) else: ips = set((n['ip'], n['port']) for n in ring_data.devs if n) return ips def get_ringmd5(self, hosts, ringfile): """ Compare ring md5sum's with those on remote host :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) :param ringfile: The local ring file to compare the md5sum with. """ stats = {} matches = 0 errors = 0 md5sum = md5() with open(ringfile, 'rb') as f: block = f.read(4096) while block: md5sum.update(block) block = f.read(4096) ring_sum = md5sum.hexdigest() recon = Scout("ringmd5", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking ring md5sums" % self._ptime() if self.verbose: print "-> On disk %s md5sum: %s" % (ringfile, ring_sum) for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: stats[url] = response[ringfile] if response[ringfile] != ring_sum: print "!! %s (%s) doesn't match on disk md5sum" % \ (url, response[ringfile]) else: matches = matches + 1 if self.verbose: print "-> %s matches." % url else: errors = errors + 1 print "%s/%s hosts matched, %s error[s] while checking hosts." \ % (matches, len(hosts), errors) print "=" * 79 def async_check(self, hosts): """ Obtain and print async pending statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ scan = {} recon = Scout("async", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking async pendings" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: scan[url] = response['async_pending'] stats = self._gen_stats(scan.values(), 'async_pending') if stats['reported'] > 0: self._print_stats(stats) else: print "[async_pending] - No hosts returned valid data." print "=" * 79 def umount_check(self, hosts): """ Check for and print unmounted drives :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ unmounted = {} errors = {} recon = Scout("unmounted", self.verbose, self.suppress_errors, self.timeout) print "[%s] Getting unmounted drives from %s hosts..." % \ (self._ptime(), len(hosts)) for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: unmounted[url] = [] errors[url] = [] for i in response: if not isinstance(i['mounted'], bool): errors[url].append(i['device']) else: unmounted[url].append(i['device']) for host in unmounted: node = urlparse(host).netloc for entry in unmounted[host]: print "Not mounted: %s on %s" % (entry, node) for host in errors: node = urlparse(host).netloc for entry in errors[host]: print "Device errors: %s on %s" % (entry, node) print "=" * 79 def expirer_check(self, hosts): """ Obtain and print expirer statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ stats = {'object_expiration_pass': [], 'expired_last_pass': []} recon = Scout("expirer/%s" % self.server_type, self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking on expirers" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: stats['object_expiration_pass'].append( response.get('object_expiration_pass')) stats['expired_last_pass'].append( response.get('expired_last_pass')) for k in stats: if stats[k]: computed = self._gen_stats(stats[k], name=k) if computed['reported'] > 0: self._print_stats(computed) else: print "[%s] - No hosts returned valid data." % k else: print "[%s] - No hosts returned valid data." % k print "=" * 79 def replication_check(self, hosts): """ Obtain and print replication statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ stats = {'replication_time': [], 'failure': [], 'success': [], 'attempted': []} recon = Scout("replication/%s" % self.server_type, self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking on replication" % self._ptime() least_recent_time = 9999999999 least_recent_url = None most_recent_time = 0 most_recent_url = None for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: stats['replication_time'].append( response.get('replication_time')) repl_stats = response['replication_stats'] if repl_stats: for stat_key in ['attempted', 'failure', 'success']: stats[stat_key].append(repl_stats.get(stat_key)) last = response.get('replication_last', 0) if last < least_recent_time: least_recent_time = last least_recent_url = url if last > most_recent_time: most_recent_time = last most_recent_url = url for k in stats: if stats[k]: if k != 'replication_time': computed = self._gen_stats(stats[k], name='replication_%s' % k) else: computed = self._gen_stats(stats[k], name=k) if computed['reported'] > 0: self._print_stats(computed) else: print "[%s] - No hosts returned valid data." % k else: print "[%s] - No hosts returned valid data." % k if least_recent_url is not None: host = urlparse(least_recent_url).netloc if not least_recent_time: print 'Oldest completion was NEVER by %s.' % host else: elapsed = time.time() - least_recent_time elapsed, elapsed_unit = seconds2timeunit(elapsed) print 'Oldest completion was %s (%d %s ago) by %s.' % ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(least_recent_time)), elapsed, elapsed_unit, host) if most_recent_url is not None: host = urlparse(most_recent_url).netloc elapsed = time.time() - most_recent_time elapsed, elapsed_unit = seconds2timeunit(elapsed) print 'Most recent completion was %s (%d %s ago) by %s.' % ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(most_recent_time)), elapsed, elapsed_unit, host) print "=" * 79 def object_replication_check(self, hosts): """ Obtain and print replication statistics from object servers :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ stats = {} recon = Scout("replication", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking on replication" % self._ptime() least_recent_time = 9999999999 least_recent_url = None most_recent_time = 0 most_recent_url = None for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: stats[url] = response['object_replication_time'] last = response.get('object_replication_last', 0) if last < least_recent_time: least_recent_time = last least_recent_url = url if last > most_recent_time: most_recent_time = last most_recent_url = url times = [x for x in stats.values() if x is not None] if len(stats) > 0 and len(times) > 0: computed = self._gen_stats(times, 'replication_time') if computed['reported'] > 0: self._print_stats(computed) else: print "[replication_time] - No hosts returned valid data." else: print "[replication_time] - No hosts returned valid data." if least_recent_url is not None: host = urlparse(least_recent_url).netloc if not least_recent_time: print 'Oldest completion was NEVER by %s.' % host else: elapsed = time.time() - least_recent_time elapsed, elapsed_unit = seconds2timeunit(elapsed) print 'Oldest completion was %s (%d %s ago) by %s.' % ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(least_recent_time)), elapsed, elapsed_unit, host) if most_recent_url is not None: host = urlparse(most_recent_url).netloc elapsed = time.time() - most_recent_time elapsed, elapsed_unit = seconds2timeunit(elapsed) print 'Most recent completion was %s (%d %s ago) by %s.' % ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(most_recent_time)), elapsed, elapsed_unit, host) print "=" * 79 def updater_check(self, hosts): """ Obtain and print updater statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ stats = [] recon = Scout("updater/%s" % self.server_type, self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking updater times" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: if response['%s_updater_sweep' % self.server_type]: stats.append(response['%s_updater_sweep' % self.server_type]) if len(stats) > 0: computed = self._gen_stats(stats, name='updater_last_sweep') if computed['reported'] > 0: self._print_stats(computed) else: print "[updater_last_sweep] - No hosts returned valid data." else: print "[updater_last_sweep] - No hosts returned valid data." print "=" * 79 def auditor_check(self, hosts): """ Obtain and print obj auditor statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ scan = {} adone = '%s_auditor_pass_completed' % self.server_type afail = '%s_audits_failed' % self.server_type apass = '%s_audits_passed' % self.server_type asince = '%s_audits_since' % self.server_type recon = Scout("auditor/%s" % self.server_type, self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking auditor stats" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: scan[url] = response if len(scan) < 1: print "Error: No hosts available" return stats = {} stats[adone] = [scan[i][adone] for i in scan if scan[i][adone] is not None] stats[afail] = [scan[i][afail] for i in scan if scan[i][afail] is not None] stats[apass] = [scan[i][apass] for i in scan if scan[i][apass] is not None] stats[asince] = [scan[i][asince] for i in scan if scan[i][asince] is not None] for k in stats: if len(stats[k]) < 1: print "[%s] - No hosts returned valid data." % k else: if k != asince: computed = self._gen_stats(stats[k], k) if computed['reported'] > 0: self._print_stats(computed) if len(stats[asince]) >= 1: low = min(stats[asince]) high = max(stats[asince]) total = sum(stats[asince]) average = total / len(stats[asince]) print '[last_pass] oldest: %s, newest: %s, avg: %s' % \ (self._ptime(low), self._ptime(high), self._ptime(average)) print "=" * 79 def nested_get_value(self, key, recon_entry): """ Generator that yields all values for given key in a recon cache entry. This is for use with object auditor recon cache entries. If the object auditor has run in 'once' mode with a subset of devices specified the checksum auditor section will have an entry of the form: {'object_auditor_stats_ALL': { 'disk1disk2diskN': {..}} The same is true of the ZBF auditor cache entry section. We use this generator to find all instances of a particular key in these multi- level dictionaries. """ for k, v in recon_entry.items(): if isinstance(v, dict): for value in self.nested_get_value(key, v): yield value if k == key: yield v def object_auditor_check(self, hosts): """ Obtain and print obj auditor statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ all_scan = {} zbf_scan = {} atime = 'audit_time' bprocessed = 'bytes_processed' passes = 'passes' errors = 'errors' quarantined = 'quarantined' recon = Scout("auditor/object", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking auditor stats " % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: if response['object_auditor_stats_ALL']: all_scan[url] = response['object_auditor_stats_ALL'] if response['object_auditor_stats_ZBF']: zbf_scan[url] = response['object_auditor_stats_ZBF'] if len(all_scan) > 0: stats = {} stats[atime] = [(self.nested_get_value(atime, all_scan[i])) for i in all_scan] stats[bprocessed] = [(self.nested_get_value(bprocessed, all_scan[i])) for i in all_scan] stats[passes] = [(self.nested_get_value(passes, all_scan[i])) for i in all_scan] stats[errors] = [(self.nested_get_value(errors, all_scan[i])) for i in all_scan] stats[quarantined] = [(self.nested_get_value(quarantined, all_scan[i])) for i in all_scan] for k in stats: if None in stats[k]: stats[k] = [x for x in stats[k] if x is not None] if len(stats[k]) < 1: print "[Auditor %s] - No hosts returned valid data." % k else: computed = self._gen_stats(stats[k], name='ALL_%s_last_path' % k) if computed['reported'] > 0: self._print_stats(computed) else: print "[ALL_auditor] - No hosts returned valid data." else: print "[ALL_auditor] - No hosts returned valid data." if len(zbf_scan) > 0: stats = {} stats[atime] = [(self.nested_get_value(atime, zbf_scan[i])) for i in zbf_scan] stats[bprocessed] = [(self.nested_get_value(bprocessed, zbf_scan[i])) for i in zbf_scan] stats[errors] = [(self.nested_get_value(errors, zbf_scan[i])) for i in zbf_scan] stats[quarantined] = [(self.nested_get_value(quarantined, zbf_scan[i])) for i in zbf_scan] for k in stats: if None in stats[k]: stats[k] = [x for x in stats[k] if x is not None] if len(stats[k]) < 1: print "[Auditor %s] - No hosts returned valid data." % k else: computed = self._gen_stats(stats[k], name='ZBF_%s_last_path' % k) if computed['reported'] > 0: self._print_stats(computed) else: print "[ZBF_auditor] - No hosts returned valid data." else: print "[ZBF_auditor] - No hosts returned valid data." print "=" * 79 def load_check(self, hosts): """ Obtain and print load average statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ load1 = {} load5 = {} load15 = {} recon = Scout("load", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking load averages" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: load1[url] = response['1m'] load5[url] = response['5m'] load15[url] = response['15m'] stats = {"1m": load1, "5m": load5, "15m": load15} for item in stats: if len(stats[item]) > 0: computed = self._gen_stats(stats[item].values(), name='%s_load_avg' % item) self._print_stats(computed) else: print "[%s_load_avg] - No hosts returned valid data." % item print "=" * 79 def quarantine_check(self, hosts): """ Obtain and print quarantine statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ objq = {} conq = {} acctq = {} recon = Scout("quarantined", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking quarantine" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: objq[url] = response['objects'] conq[url] = response['containers'] acctq[url] = response['accounts'] stats = {"objects": objq, "containers": conq, "accounts": acctq} for item in stats: if len(stats[item]) > 0: computed = self._gen_stats(stats[item].values(), name='quarantined_%s' % item) self._print_stats(computed) else: print "No hosts returned valid data." print "=" * 79 def socket_usage(self, hosts): """ Obtain and print /proc/net/sockstat statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ inuse4 = {} mem = {} inuse6 = {} timewait = {} orphan = {} recon = Scout("sockstat", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking socket usage" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: inuse4[url] = response['tcp_in_use'] mem[url] = response['tcp_mem_allocated_bytes'] inuse6[url] = response.get('tcp6_in_use', 0) timewait[url] = response['time_wait'] orphan[url] = response['orphan'] stats = {"tcp_in_use": inuse4, "tcp_mem_allocated_bytes": mem, "tcp6_in_use": inuse6, "time_wait": timewait, "orphan": orphan} for item in stats: if len(stats[item]) > 0: computed = self._gen_stats(stats[item].values(), item) self._print_stats(computed) else: print "No hosts returned valid data." print "=" * 79 def disk_usage(self, hosts, top=0, human_readable=False): """ Obtain and print disk usage statistics :param hosts: set of hosts to check. in the format of: set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) """ stats = {} highs = [] lows = [] raw_total_used = [] raw_total_avail = [] percents = {} top_percents = [(None, 0)] * top recon = Scout("diskusage", self.verbose, self.suppress_errors, self.timeout) print "[%s] Checking disk usage now" % self._ptime() for url, response, status in self.pool.imap(recon.scout, hosts): if status == 200: hostusage = [] for entry in response: if not isinstance(entry['mounted'], bool): print "-> %s/%s: Error: %s" % (url, entry['device'], entry['mounted']) elif entry['mounted']: used = float(entry['used']) / float(entry['size']) \ * 100.0 raw_total_used.append(entry['used']) raw_total_avail.append(entry['avail']) hostusage.append(round(used, 2)) for ident, oused in top_percents: if oused < used: top_percents.append( (url + ' ' + entry['device'], used)) top_percents.sort(key=lambda x: -x[1]) top_percents.pop() break stats[url] = hostusage for url in stats: if len(stats[url]) > 0: # get per host hi/los for another day low = min(stats[url]) high = max(stats[url]) highs.append(high) lows.append(low) for percent in stats[url]: percents[int(percent)] = percents.get(int(percent), 0) + 1 else: print "-> %s: Error. No drive info available." % url if len(lows) > 0: low = min(lows) high = max(highs) # dist graph shamelessly stolen from https://github.com/gholt/tcod print "Distribution Graph:" mul = 69.0 / max(percents.values()) for percent in sorted(percents): print '% 3d%%%5d %s' % (percent, percents[percent], '*' * int(percents[percent] * mul)) raw_used = sum(raw_total_used) raw_avail = sum(raw_total_avail) raw_total = raw_used + raw_avail avg_used = 100.0 * raw_used / raw_total if human_readable: raw_used = size_suffix(raw_used) raw_avail = size_suffix(raw_avail) raw_total = size_suffix(raw_total) print "Disk usage: space used: %s of %s" % (raw_used, raw_total) print "Disk usage: space free: %s of %s" % (raw_avail, raw_total) print "Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" % \ (low, high, avg_used) else: print "No hosts returned valid data." print "=" * 79 if top_percents: print 'TOP %s' % top for ident, used in top_percents: if ident: url, device = ident.split() host = urlparse(url).netloc.split(':')[0] print '%.02f%% %s' % (used, '%-15s %s' % (host, device)) def main(self): """ Retrieve and report cluster info from hosts running recon middleware. """ print "=" * 79 usage = ''' usage: %prog [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat] [--human-readable] \taccount|container|object Defaults to object server. ex: %prog container -l --auditor ''' args = optparse.OptionParser(usage) args.add_option('--verbose', '-v', action="store_true", help="Print verbose info") args.add_option('--suppress', action="store_true", help="Suppress most connection related errors") args.add_option('--async', '-a', action="store_true", help="Get async stats") args.add_option('--replication', '-r', action="store_true", help="Get replication stats") args.add_option('--auditor', action="store_true", help="Get auditor stats") args.add_option('--updater', action="store_true", help="Get updater stats") args.add_option('--expirer', action="store_true", help="Get expirer stats") args.add_option('--unmounted', '-u', action="store_true", help="Check cluster for unmounted devices") args.add_option('--diskusage', '-d', action="store_true", help="Get disk usage stats") args.add_option('--human-readable', action="store_true", help="Use human readable suffix for disk usage stats") args.add_option('--loadstats', '-l', action="store_true", help="Get cluster load average stats") args.add_option('--quarantined', '-q', action="store_true", help="Get cluster quarantine stats") args.add_option('--md5', action="store_true", help="Get md5sum of servers ring and compare to " "local copy") args.add_option('--sockstat', action="store_true", help="Get cluster socket usage stats") args.add_option('--top', type='int', metavar='COUNT', default=0, help='Also show the top COUNT entries in rank order.') args.add_option('--all', action="store_true", help="Perform all checks. Equal to -arudlq --md5 " "--sockstat") args.add_option('--zone', '-z', type="int", help="Only query servers in specified zone") args.add_option('--timeout', '-t', type="int", metavar="SECONDS", help="Time to wait for a response from a server", default=5) args.add_option('--swiftdir', default="/etc/swift", help="Default = /etc/swift") options, arguments = args.parse_args() if len(sys.argv) <= 1 or len(arguments) > 1: args.print_help() sys.exit(0) if arguments: if arguments[0] in self.check_types: self.server_type = arguments[0] else: print "Invalid Server Type" args.print_help() sys.exit(1) else: self.server_type = 'object' swift_dir = options.swiftdir ring_file = os.path.join(swift_dir, '%s.ring.gz' % self.server_type) self.verbose = options.verbose self.suppress_errors = options.suppress self.timeout = options.timeout if options.zone is not None: hosts = self.get_devices(options.zone, swift_dir, self.server_type) else: hosts = self.get_devices(None, swift_dir, self.server_type) print "--> Starting reconnaissance on %s hosts" % len(hosts) print "=" * 79 if options.all: if self.server_type == 'object': self.async_check(hosts) self.object_replication_check(hosts) self.object_auditor_check(hosts) self.updater_check(hosts) self.expirer_check(hosts) elif self.server_type == 'container': self.replication_check(hosts) self.auditor_check(hosts) self.updater_check(hosts) elif self.server_type == 'account': self.replication_check(hosts) self.auditor_check(hosts) self.umount_check(hosts) self.load_check(hosts) self.disk_usage(hosts) self.get_ringmd5(hosts, ring_file) self.quarantine_check(hosts) self.socket_usage(hosts) else: if options.async: if self.server_type == 'object': self.async_check(hosts) else: print "Error: Can't check asyncs on non object servers." if options.unmounted: self.umount_check(hosts) if options.replication: if self.server_type == 'object': self.object_replication_check(hosts) else: self.replication_check(hosts) if options.auditor: if self.server_type == 'object': self.object_auditor_check(hosts) else: self.auditor_check(hosts) if options.updater: if self.server_type == 'account': print "Error: Can't check updaters on account servers." else: self.updater_check(hosts) if options.expirer: if self.server_type == 'object': self.expirer_check(hosts) else: print "Error: Can't check expired on non object servers." if options.loadstats: self.load_check(hosts) if options.diskusage: self.disk_usage(hosts, options.top, options.human_readable) if options.md5: self.get_ringmd5(hosts, ring_file) if options.quarantined: self.quarantine_check(hosts) if options.sockstat: self.socket_usage(hosts) def main(): try: reconnoiter = SwiftRecon() reconnoiter.main() except KeyboardInterrupt: print '\n' if __name__ == '__main__': main() swift-1.13.1/swift/cli/__init__.py0000664000175400017540000000000012323703611020067 0ustar jenkinsjenkins00000000000000swift-1.13.1/swift/__init__.py0000664000175400017540000000305312323703611017333 0ustar jenkinsjenkins00000000000000# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import os import gettext import pkg_resources try: # First, try to get our version out of PKG-INFO. If we're installed, # this'll let us find our version without pulling in pbr. After all, if # we're installed on a system, we're not in a Git-managed source tree, so # pbr doesn't really buy us anything. __version__ = __canonical_version__ = pkg_resources.get_provider( pkg_resources.Requirement.parse('swift')).version except pkg_resources.DistributionNotFound: # No PKG-INFO? We're probably running from a checkout, then. Let pbr do # its thing to figure out a version number. import pbr.version _version_info = pbr.version.VersionInfo('swift') __version__ = _version_info.release_string() __canonical_version__ = _version_info.version_string() _localedir = os.environ.get('SWIFT_LOCALEDIR') _t = gettext.translation('swift', localedir=_localedir, fallback=True) def gettext_(msg): return _t.gettext(msg) swift-1.13.1/.mailmap0000664000175400017540000000644712323703611015521 0ustar jenkinsjenkins00000000000000Greg Holt gholt Greg Holt gholt Greg Holt gholt Greg Holt gholt Greg Holt Greg Holt John Dickinson Michael Barton Michael Barton Michael Barton Mike Barton Clay Gerrard Clay Gerrard Clay Gerrard clayg David Goetz David Goetz Anne Gentle Anne Gentle annegentle Fujita Tomonori Greg Lange Greg Lange Chmouel Boudjnah Gaurav B. Gangalwar gaurav@gluster.com <> Joe Arnold Kapil Thangavelu kapil.foss@gmail.com <> Samuel Merritt Morita Kazutaka Zhongyue Luo Russ Nelson Marcelo Martins Andrew Clay Shafer Soren Hansen Soren Hansen Ye Jia Xu monsterxx03 Victor Rodionov Florian Hines Jay Payne Doug Weimer Li Riqiang lrqrun Cory Wright Julien Danjou David Hadas Yaguang Wang ywang19 Liu Siqi dk647 James E. Blair Kun Huang Michael Shuler Ilya Kharin Dmitry Ukov Ukov Dmitry Tom Fifield Tom Fifield Sascha Peilicke Sascha Peilicke Zhenguo Niu Peter Portante Christian Schwede Constantine Peresypkin Madhuri Kumari madhuri swift-1.13.1/test-requirements.txt0000664000175400017540000000027212323703611020327 0ustar jenkinsjenkins00000000000000# Hacking already pins down pep8, pyflakes and flake8 hacking>=0.8.0,<0.9 coverage nose nosexcover openstack.nose_plugin nosehtmloutput sphinx>=1.1.2,<1.2 mock>=0.8.0 python-swiftclient swift-1.13.1/.coveragerc0000664000175400017540000000014512323703611016206 0ustar jenkinsjenkins00000000000000[run] branch = True omit = /usr*,setup.py,*egg*,.venv/*,.tox/*,test/* [report] ignore-errors = True swift-1.13.1/CONTRIBUTING.md0000664000175400017540000000213112323703611016313 0ustar jenkinsjenkins00000000000000If you would like to contribute to the development of OpenStack, you must follow the steps in the "If you're a developer, start here" section of this page: [http://wiki.openstack.org/HowToContribute](http://wiki.openstack.org/HowToContribute#If_you.27re_a_developer.2C_start_here:) Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at [http://wiki.openstack.org/GerritWorkflow](http://wiki.openstack.org/GerritWorkflow). Gerrit is the review system used in the OpenStack projects. We're sorry, but we won't be able to respond to pull requests submitted through GitHub. Bugs should be filed [on Launchpad](https://bugs.launchpad.net/swift), not in GitHub's issue tracker. Recommended workflow ==================== * Set up a [Swift All-In-One VM](http://docs.openstack.org/developer/swift/development_saio.html). * Make your changes. * Run unit tests, functional tests, probe tests ``./.unittests`` ``./.functests`` ``./.probetests`` * Run ``tox`` (no command-line args needed) * ``git review`` swift-1.13.1/doc/0000775000175400017540000000000012323703665014643 5ustar jenkinsjenkins00000000000000swift-1.13.1/doc/manpages/0000775000175400017540000000000012323703665016436 5ustar jenkinsjenkins00000000000000swift-1.13.1/doc/manpages/swift-object-info.10000664000175400017540000000322712323703611022044 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-info 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-info \- Openstack-swift object-info tool .SH SYNOPSIS .LP .B swift-object-info [OBJECT_FILE] [SWIFT_DIR] .SH DESCRIPTION .PP This is a very simple swift tool that allows a swiftop engineer to retrieve information about an object that is located on the storage node. One calls the tool with a given object file as it is stored on the storage node system. It will then return several information about that object such as; .PD 0 .IP "- Account it belongs to" .IP "- Container " .IP "- Object hash " .IP "- Content Type " .IP "- timestamp " .IP "- Etag " .IP "- Content Length " .IP "- User Metadata " .IP "- Location on the ring " .PD .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR swift-account-info(1), .BR swift-container-info(1), .BR swift-get-nodes(1) swift-1.13.1/doc/manpages/swift-object-updater.10000664000175400017540000000500212323703611022546 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-updater 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-updater \- Openstack-swift object updater .SH SYNOPSIS .LP .B swift-object-updater [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP The object updater is responsible for updating object information in container listings. It will check to see if there are any locally queued updates on the filesystem of each devices, what is also known as async pending file(s), walk each one and update the container listing. For example, suppose a container server is under load and a new object is put into the system. The object will be immediately available for reads as soon as the proxy server responds to the client with success. However, the object server has not been able to update the object listing in the container server. Therefore, the update would be queued locally for a later update. Container listings, therefore, may not immediately contain the object. This is where an eventual consistency window will most likely come in to play. In practice, the consistency window is only as large as the frequency at which the updater runs and may not even be noticed as the proxy server will route listing requests to the first container server which responds. The server under load may not be the one that serves subsequent listing requests – one of the other two replicas may handle the listing. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-updater and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR object-server.conf(5) swift-1.13.1/doc/manpages/account-server.conf.50000664000175400017540000002100512323703611022375 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH account-server.conf 5 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B account-server.conf \- configuration file for the openstack-swift account server .SH SYNOPSIS .LP .B account-server.conf .SH DESCRIPTION .PP This is the configuration file used by the account server and other account background services, such as; replicator, auditor and reaper. The configuration file follows the python-pastedeploy syntax. The file is divided into sections, which are enclosed by square brackets. Each section will contain a certain number of key/value parameters which are described later. Any line that begins with a '#' symbol is ignored. You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION .PD 1 .RS 0 This is indicated by section named [DEFAULT]. Below are the parameters that are acceptable within this section. .IP "\fBbind_ip\fR" IP address the account server should bind to. The default is 0.0.0.0 which will make it bind to all available addresses. .IP "\fBbind_port\fR" TCP port the account server should bind to. The default is 6002. .IP \fBbacklog\fR TCP backlog. Maximum number of allowed pending connections. The default value is 4096. .IP \fBworkers\fR The number of pre-forked processes that will accept connections. Zero means no fork. The default is auto which will make the server try to match the number of effective cpu cores if python multiprocessing is available (included with most python distributions >= 2.6) or fallback to one. It's worth noting that individual workers will use many eventlet co-routines to service multiple concurrent requests. .IP \fBmax_clients\fR Maximum number of clients one worker can process simultaneously (it will actually accept(2) N + 1). Setting this to one (1) will only handle one request at a time, without accepting another request concurrently. The default is 1024. .IP \fBuser\fR The system user that the account server will run as. The default is swift. .IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. .IP \fBdevices\fR Parent directory or where devices are mounted. Default is /srv/node. .IP \fBmount_check\fR Whether or not check if the devices are mounted to prevent accidentally writing to the root device. The default is set to true. .IP \fBlog_name\fR Label used when logging. The default is swift. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .RE .PD .SH PIPELINE SECTION .PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that are acceptable within this section. .IP "\fBpipeline\fR" It is used when you need apply a number of filters. It is a list of filters ended by an application. The normal pipeline is "healthcheck recon account-server". .RE .PD .SH APP SECTION .PD 1 .RS 0 This is indicated by section name [app:account-server]. Below are the parameters that are acceptable within this section. .IP "\fBuse\fR" Entry point for paste.deploy for the account server. This is the reference to the installed python egg. This is normally \fBegg:swift#account\fR. .IP "\fBset log_name\fR Label used when logging. The default is account-server. .IP "\fBset log_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP "\fB set log_level\fR Logging level. The default is INFO. .IP "\fB set log_requests\fR Enables request logging. The default is True. .IP "\fB set log_address\fR Logging address. The default is /dev/log. .RE .PD .SH FILTER SECTION .PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. Below are the filters available and respective acceptable parameters. .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#healthcheck\fR. .IP "\fBdisable_path\fR" An optional filesystem path which, if present, will cause the healthcheck URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE". .RE .RS 0 .IP "\fB[filter:recon]\fR" .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the recon middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#recon\fR. .IP "\fBrecon_cache_path\fR" The recon_cache_path simply sets the directory where stats for a few items will be stored. Depending on the method of deployment you may need to create this directory manually and ensure that swift has read/write. The default is /var/cache/swift. .RE .PD .SH ADDITIONAL SECTIONS .PD 1 .RS 0 The following sections are used by other swift-account services, such as replicator, auditor and reaper. .IP "\fB[account-replicator]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is account-replicator. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBper_diff\fR The default is 1000. .IP \fBmax_diffs\fR This caps how long the replicator will spend trying to sync a given database per pass so the other databases don't get starved. The default is 100. .IP \fBconcurrency\fR Number of replication workers to spawn. The default is 8. .IP "\fBrun_pause [deprecated]\fR" Time in seconds to wait between replication passes. The default is 10. .IP \fBinterval\fR Replaces run_pause with the more standard "interval", which means the replicator won't pause unless it takes less than the interval set. The default is 30. .IP \fBerror_suppression_interval\fR How long without an error before a node's error count is reset. This will also be how long before a node is re-enabled after suppression is triggered. The default is 60 seconds. .IP \fBerror_suppression_limit\fR How many errors can accumulate before a node is temporarily ignored. The default is 10 seconds. .IP \fBnode_timeout\fR Request timeout to external services. The default is 10 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .IP \fBreclaim_age\fR Time elapsed in seconds before an account can be reclaimed. The default is 604800 seconds. .RE .RS 0 .IP "\fB[account-auditor]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is account-auditor. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 account per device per interval. The default is 1800 seconds. .IP \fBaccounts_per_second\fR Maximum accounts audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200. .RE .RS 0 .IP "\fB[account-reaper]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is account-reaper. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBconcurrency\fR Number of reaper workers to spawn. The default is 25. .IP \fBinterval\fR Minimum time for a pass to take. The default is 3600 seconds. .IP \fBnode_timeout\fR Request timeout to external services. The default is 10 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-account-server and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-account-server(1), swift-1.13.1/doc/manpages/swift-container-replicator.10000664000175400017540000000422212323703611023765 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-replicator 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-replicator \- Openstack-swift container replicator .SH SYNOPSIS .LP .B swift-container-replicator [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP Replication is designed to keep the system in a consistent state in the face of temporary error conditions like network outages or drive failures. The replication processes compare local data with each remote copy to ensure they all contain the latest version. Container replication uses a combination of hashes and shared high water marks to quickly compare subsections of each partition. .PP Replication updates are push based. Container replication push missing records over HTTP or rsync whole database files. The replicator also ensures that data is removed from the system. When an container item is deleted a tombstone is set as the latest version of the item. The replicator will see the tombstone and ensure that the item is removed from the entire system. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-container-replicator and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR container-server.conf(5) swift-1.13.1/doc/manpages/swift-container-sync.10000664000175400017540000000361012323703611022575 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-sync 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-sync \- Openstack-swift container sync .SH SYNOPSIS .LP .B swift-container-sync [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP Swift has a feature where all the contents of a container can be mirrored to another container through background synchronization. Swift cluster operators configure their cluster to allow/accept sync requests to/from other clusters, and the user specifies where to sync their container to along with a secret synchronization key. .PP The swift-container-sync does the job of sending updates to the remote container. This is done by scanning the local devices for container databases and checking for x-container-sync-to and x-container-sync-key metadata values. If they exist, newer rows since the last sync will trigger PUTs or DELETEs to the other container. .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-container-sync and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/overview_container_sync.html and .BI http://docs.openstack.org .LP .SH "SEE ALSO" .BR container-server.conf(5) swift-1.13.1/doc/manpages/swift-account-info.10000664000175400017540000000317612323703611022235 0ustar jenkinsjenkins00000000000000.\" .\" Author: Madhuri Kumari .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-account-info 1 "3/22/2014" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-account-info \- Openstack-swift account-info tool .SH SYNOPSIS .LP .B swift-account-info [ACCOUNT_DB_FILE] [SWIFT_DIR] .SH DESCRIPTION .PP This is a very simple swift tool that allows a swiftop engineer to retrieve information about an account that is located on the storage node. One calls the tool with a given db file as it is stored on the storage node system. It will then return several information about that account such as; .PD 0 .IP "- Account" .IP "- Account hash " .IP "- Created timestamp " .IP "- Put timestamp " .IP "- Delete timestamp " .IP "- Container Count " .IP "- Object count " .IP "- Bytes used " .IP "- Chexor " .IP "- ID" .IP "- User Metadata " .IP "- Ring Location" .PD .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR swift-container-info(1), .BR swift-get-nodes(1), .BR swift-object-info(1) swift-1.13.1/doc/manpages/swift-init.10000664000175400017540000000721212323703611020606 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-init 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-init \- Openstack-swift swift-init tool .SH SYNOPSIS .LP .B swift-init [ ...] [options] .SH DESCRIPTION .PP The swift-init tool can be used to initialize all swift daemons available as part of openstack-swift. Instead of calling individual init scripts for each swift daemon, one can just use swift-init. With swift-init you can initialize just one swift service, such as the "proxy", or a combination of them. The tool also allows one to use the keywords such as "all", "main" and "rest" for the argument. \fBServers:\fR .PD 0 .RS 4 .IP "\fIproxy\fR" "4" .IP " - Initializes the swift proxy daemon" .RE .RS 4 .IP "\fIobject\fR, \fIobject-replicator\fR, \fIobject-auditor\fR, \fIobject-updater\fR" .IP " - Initializes the swift object daemons above" .RE .RS 4 .IP "\fIcontainer\fR, \fIcontainer-update\fR, \fIcontainer-replicator\fR, \fIcontainer-auditor\fR" .IP " - Initializes the swift container daemons above" .RE .RS 4 .IP "\fIaccount\fR, \fIaccount-auditor\fR, \fIaccount-reaper\fR, \fIaccount-replicator\fR" .IP " - Initializes the swift account daemons above" .RE .RS 4 .IP "\fIall\fR" .IP " - Initializes \fBall\fR the swift daemons" .RE .RS 4 .IP "\fImain\fR" .IP " - Initializes all the \fBmain\fR swift daemons" .IP " (proxy, container, account and object servers)" .RE .RS 4 .IP "\fIrest\fR" .IP " - Initializes all the other \fBswift background daemons\fR" .IP " (updater, replicator, auditor, reaper, etc)" .RE .PD \fBCommands:\fR .RS 4 .PD 0 .IP "\fIforce-reload\fR: \t\t alias for reload" .IP "\fIno-daemon\fR: \t\t start a server interactively" .IP "\fIno-wait\fR: \t\t\t spawn server and return immediately" .IP "\fIonce\fR: \t\t\t start server and run one pass on supporting daemons" .IP "\fIreload\fR: \t\t\t graceful shutdown then restart on supporting servers" .IP "\fIrestart\fR: \t\t\t stops then restarts server" .IP "\fIshutdown\fR: \t\t allow current requests to finish on supporting servers" .IP "\fIstart\fR: \t\t\t starts a server" .IP "\fIstatus\fR: \t\t\t display status of tracked pids for server" .IP "\fIstop\fR: \t\t\t stops a server" .PD .RE \fBOptions:\fR .RS 4 .PD 0 .IP "-h, --help \t\t\t show this help message and exit" .IP "-v, --verbose \t\t\t display verbose output" .IP "-w, --no-wait \t\t\t won't wait for server to start before returning .IP "-o, --once \t\t\t only run one pass of daemon .IP "-n, --no-daemon \t\t start server interactively .IP "-g, --graceful \t\t send SIGHUP to supporting servers .IP "-c N, --config-num=N \t send command to the Nth server only .IP "-k N, --kill-wait=N \t wait N seconds for processes to die (default 15) .IP "-r RUN_DIR, --run-dir=RUN_DIR directory where the pids will be stored (default /var/run/swift) .PD .RE .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html swift-1.13.1/doc/manpages/swift-object-auditor.10000664000175400017540000000336012323703611022556 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-auditor 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-auditor \- Openstack-swift object auditor .SH SYNOPSIS .LP .B swift-object-auditor [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] [-z|--zero_byte_fps] .SH DESCRIPTION .PP The object auditor crawls the local object system checking the integrity of objects. If corruption is found (in the case of bit rot, for example), the file is quarantined, and replication will replace the bad file from another replica. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .IP "-z ZERO_BYTE_FPS" .IP "--zero_byte_fps=ZERO_BYTE_FPS" .RS 4 .IP "Audit only zero byte files at specified files/sec" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-auditor and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR object-server.conf(5) swift-1.13.1/doc/manpages/swift-account-server.10000664000175400017540000000260412323703611022603 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-account-server 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-account-server \- Openstack-swift account server .SH SYNOPSIS .LP .B swift-account-server [CONFIG] [-h|--help] [-v|--verbose] .SH DESCRIPTION .PP The Account Server's primary job is to handle listings of containers. The listings are stored as sqlite database files, and replicated across the cluster similar to how objects are. .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-account-server and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html and .BI http://docs.openstack.org .SH "SEE ALSO" .BR account-server.conf(5) swift-1.13.1/doc/manpages/swift-orphans.10000664000175400017540000000361212323703611021315 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-orphans 1 "3/15/2012" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-orphans \- Openstack-swift orphans tool .SH SYNOPSIS .LP .B swift-orphans [-h|--help] [-a|--age] [-k|--kill] [-w|--wide] [-r|--run-dir] .SH DESCRIPTION .PP Lists and optionally kills orphaned Swift processes. This is done by scanning /var/run/swift or the directory specified to the \-r switch for .pid files and listing any processes that look like Swift processes but aren't associated with the pids in those .pid files. Any Swift processes running with the 'once' parameter are ignored, as those are usually for full-speed audit scans and such. Example (sends SIGTERM to all orphaned Swift processes older than two hours): swift-orphans \-a 2 \-k TERM The options are as follows: .RS 4 .PD 0 .IP "-a HOURS" .IP "--age=HOURS" .RS 4 .IP "Look for processes at least HOURS old; default: 24" .RE .IP "-k SIGNAL" .IP "--kill=SIGNAL" .RS 4 .IP "Send SIGNAL to matched processes; default: just list process information" .RE .IP "-w" .IP "--wide" .RS 4 .IP "Don't clip the listing at 80 characters" .RE .PD .RE .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html swift-1.13.1/doc/manpages/swift-get-nodes.10000664000175400017540000000537712323703611021542 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-get-nodes 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-get-nodes \- Openstack-swift get-nodes tool .SH SYNOPSIS .LP .B swift-get-nodes \ [] [] .SH DESCRIPTION .PP The swift-get-nodes tool can be used to find out the location where a particular account, container or object item is located within the swift cluster nodes. For example, if you have the account hash and a container name that belongs to that account, you can use swift-get-nodes to lookup where the container resides by using the container ring. .RS 0 .IP "\fIExample:\fR" .RE .RS 4 .PD 0 .IP "$ swift-get-nodes /etc/swift/account.ring.gz MyAccount-12ac01446be2" .PD 0 .IP "Account MyAccount-12ac01446be2" .IP "Container None" .IP "Object None" .IP "Partition 221082" .IP "Hash d7e6ba68cfdce0f0e4ca7890e46cacce" .IP "Server:Port Device 172.24.24.29:6002 sdd" .IP "Server:Port Device 172.24.24.27:6002 sdr" .IP "Server:Port Device 172.24.24.32:6002 sde" .IP "Server:Port Device 172.24.24.26:6002 sdv [Handoff]" .IP "curl -I -XHEAD http://172.24.24.29:6002/sdd/221082/MyAccount-12ac01446be2" .IP "curl -I -XHEAD http://172.24.24.27:6002/sdr/221082/MyAccount-12ac01446be2" .IP "curl -I -XHEAD http://172.24.24.32:6002/sde/221082/MyAccount-12ac01446be2" .IP "curl -I -XHEAD http://172.24.24.26:6002/sdv/221082/MyAccount-12ac01446be2 # [Handoff]" .IP "ssh 172.24.24.29 ls -lah /srv/node/sdd/accounts/221082/cce/d7e6ba68cfdce0f0e4ca7890e46cacce/ " .IP "ssh 172.24.24.27 ls -lah /srv/node/sdr/accounts/221082/cce/d7e6ba68cfdce0f0e4ca7890e46cacce/" .IP "ssh 172.24.24.32 ls -lah /srv/node/sde/accounts/221082/cce/d7e6ba68cfdce0f0e4ca7890e46cacce/" .IP "ssh 172.24.24.26 ls -lah /srv/node/sdv/accounts/221082/cce/d7e6ba68cfdce0f0e4ca7890e46cacce/ # [Handoff] " .PD .RE .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR swift-account-info(1), .BR swift-container-info(1), .BR swift-object-info(1), .BR swift-ring-builder(1) swift-1.13.1/doc/manpages/swift-object-server.10000664000175400017540000000401112323703611022407 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-server 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-server \- Openstack-swift object server. .SH SYNOPSIS .LP .B swift-object-server [CONFIG] [-h|--help] [-v|--verbose] .SH DESCRIPTION .PP The Object Server is a very simple blob storage server that can store, retrieve and delete objects stored on local devices. Objects are stored as binary files on the filesystem with metadata stored in the file's extended attributes (xattrs). This requires that the underlying filesystem choice for object servers support xattrs on files. Some filesystems, like ext3, have xattrs turned off by default. Each object is stored using a path derived from the object name's hash and the operation's timestamp. Last write always wins, and ensures that the latest object version will be served. A deletion is also treated as a version of the file (a 0 byte file ending with ".ts", which stands for tombstone). This ensures that deleted files are replicated correctly and older versions don't magically reappear due to failure scenarios. .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-server and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html and .BI http://docs.openstack.org .SH "SEE ALSO" .BR object-server.conf(5) swift-1.13.1/doc/manpages/object-expirer.conf.50000664000175400017540000001002512323703611022357 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH object-expirer.conf 5 "03/15/2012" "Linux" "OpenStack Swift" .SH NAME .LP .B object-expirer.conf \- configuration file for the openstack-swift object exprier daemon .SH SYNOPSIS .LP .B object-expirer.conf .SH DESCRIPTION .PP This is the configuration file used by the object expirer daemon. The daemon's function is to query the internal hidden expiring_objects_account to discover objects that need to be deleted and to then delete them. The configuration file follows the python-pastedeploy syntax. The file is divided into sections, which are enclosed by square brackets. Each section will contain a certain number of key/value parameters which are described later. Any line that begins with a '#' symbol is ignored. You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION .PD 1 .RS 0 This is indicated by section named [DEFAULT]. Below are the parameters that are acceptable within this section. .IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. .IP \fBuser\fR The system user that the object server will run as. The default is swift. .IP \fBlog_name\fR Label used when logging. The default is swift. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .RE .PD .SH PIPELINE SECTION .PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that are acceptable within this section. .IP "\fBpipeline\fR" It is used when you need to apply a number of filters. It is a list of filters ended by an application. The default should be \fB"catch_errors cache proxy-server"\fR .RE .PD .SH APP SECTION .PD 1 .RS 0 This is indicated by section name [app:object-server]. Below are the parameters that are acceptable within this section. .IP "\fBuse\fR" Entry point for paste.deploy for the object server. This is the reference to the installed python egg. The default is \fBegg:swift#proxy\fR. See proxy-server.conf-sample for options or See proxy-server.conf manpage. .RE .PD .SH FILTER SECTION .PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. Below are the filters available and respective acceptable parameters. .RS 0 .IP "\fB[filter:cache]\fR" .RE Caching middleware that manages caching in swift. .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the memcache middleware. This is the reference to the installed python egg. The default is \fBegg:swift#memcache\fR. See proxy-server.conf-sample for options or See proxy-server.conf manpage. .RE .RS 0 .IP "\fB[filter:catch_errors]\fR" .RE .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the catch_errors middleware. This is the reference to the installed python egg. The default is \fBegg:swift#catch_errors\fR. See proxy-server.conf-sample for options or See proxy-server.conf manpage. .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-object-expirer and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-proxy-server.conf(5), swift-1.13.1/doc/manpages/proxy-server.conf.50000664000175400017540000005155112323703611022133 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH proxy-server.conf 5 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B proxy-server.conf \- configuration file for the openstack-swift proxy server .SH SYNOPSIS .LP .B proxy-server.conf .SH DESCRIPTION .PP This is the configuration file used by the proxy server and other proxy middlewares. The configuration file follows the python-pastedeploy syntax. The file is divided into sections, which are enclosed by square brackets. Each section will contain a certain number of key/value parameters which are described later. Any line that begins with a '#' symbol is ignored. You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION .PD 1 .RS 0 This is indicated by section named [DEFAULT]. Below are the parameters that are acceptable within this section. .IP "\fBbind_ip\fR" IP address the proxy server should bind to. The default is 0.0.0.0 which will make it bind to all available addresses. .IP "\fBbind_port\fR" TCP port the proxy server should bind to. The default is 80. .IP \fBbacklog\fR TCP backlog. Maximum number of allowed pending connections. The default value is 4096. .IP \fBworkers\fR The number of pre-forked processes that will accept connections. Zero means no fork. The default is auto which will make the server try to match the number of effective cpu cores if python multiprocessing is available (included with most python distributions >= 2.6) or fallback to one. It's worth noting that individual workers will use many eventlet co-routines to service multiple concurrent requests. .IP \fBmax_clients\fR Maximum number of clients one worker can process simultaneously (it will actually accept(2) N + 1). Setting this to one (1) will only handle one request at a time, without accepting another request concurrently. The default is 1024. .IP \fBuser\fR The system user that the proxy server will run as. The default is swift. .IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. .IP \fBcert_file\fR Location of the SSL certificate file. The default path is /etc/swift/proxy.crt. This is disabled by default. .IP \fBkey_file\fR Location of the SSL certificate key file. The default path is /etc/swift/proxy.key. This is disabled by default. .IP \fBlog_name\fR Label used when logging. The default is swift. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBtrans_id_suffix\fR This optional suffix (default is empty) that would be appended to the swift transaction id allows one to easily figure out from which cluster that X-Trans-Id belongs to. This is very useful when one is managing more than one swift cluster. .RE .PD .SH PIPELINE SECTION .PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that are acceptable within this section. .IP "\fBpipeline\fR" It is used when you need apply a number of filters. It is a list of filters ended by an application. The normal pipeline is "catch_errors healthcheck cache ratelimit tempauth proxy-logging proxy-server". .RE .PD .SH FILTER SECTION .PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. Below are the filters available and respective acceptable parameters. .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#healthcheck\fR. .IP "\fBdisable_path\fR" An optional filesystem path which, if present, will cause the healthcheck URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE". .RE .RS 0 .IP "\fB[filter:tempauth]\fR" .RE .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the tempauth middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#tempauth\fR. .IP "\fBset log_name\fR" Label used when logging. The default is tempauth. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBreseller_prefix\fR The reseller prefix will verify a token begins with this prefix before even attempting to validate it. Also, with authorization, only Swift storage accounts with this prefix will be authorized by this middleware. Useful if multiple auth systems are in use for one Swift cluster. The default is AUTH. .IP \fBauth_prefix\fR The auth prefix will cause requests beginning with this prefix to be routed to the auth subsystem, for granting tokens, etc. The default is /auth/. .IP \fBtoken_life\fR This is the time in seconds before the token expires. The default is 86400. .IP \fBuser__\fR Lastly, you need to list all the accounts/users you want here. The format is: user__ = [group] [group] [...] [storage_url] There are special groups of: \fI.reseller_admin\fR who can do anything to any account for this auth and also \fI.admin\fR who can do anything within the account. If neither of these groups are specified, the user can only access containers that have been explicitly allowed for them by a \fI.admin\fR or \fI.reseller_admin\fR. The trailing optional storage_url allows you to specify an alternate url to hand back to the user upon authentication. If not specified, this defaults to \fIhttp[s]://:/v1/_\fR where http or https depends on whether cert_file is specified in the [DEFAULT] section, and are based on the [DEFAULT] section's bind_ip and bind_port (falling back to 127.0.0.1 and 8080), is from this section, and is from the user__ name. Here are example entries, required for running the tests: .RE .PD 0 .RS 10 .IP "user_admin_admin = admin .admin .reseller_admin" .IP "user_test_tester = testing .admin" .IP "user_test2_tester2 = testing2 .admin" .IP "user_test_tester3 = testing3" .RE .PD .RS 0 .IP "\fB[filter:cache]\fR" .RE Caching middleware that manages caching in swift. .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the memcache middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#memcache\fR. .IP "\fBset log_name\fR" Label used when logging. The default is memcache. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBmemcache_servers\fR If not set in the configuration file, the value for memcache_servers will be read from /etc/swift/memcache.conf (see memcache.conf-sample) or lacking that file, it will default to 127.0.0.1:11211. You can specify multiple servers separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211. .IP \fBmemcache_serialization_support\fR This sets how memcache values are serialized and deserialized: .RE .PD 0 .RS 10 .IP "0 = older, insecure pickle serialization" .IP "1 = json serialization but pickles can still be read (still insecure)" .IP "2 = json serialization only (secure and the default)" .RE .RS 10 To avoid an instant full cache flush, existing installations should upgrade with 0, then set to 1 and reload, then after some time (24 hours) set to 2 and reload. In the future, the ability to use pickle serialization will be removed. If not set in the configuration file, the value for memcache_serialization_support will be read from /etc/swift/memcache.conf if it exists (see memcache.conf-sample). Otherwise, the default value as indicated above will be used. .RE .RS 0 .IP "\fB[filter:ratelimit]\fR" .RE Rate limits requests on both an Account and Container level. Limits are configurable. .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the ratelimit middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#ratelimit\fR. .IP "\fBset log_name\fR" Label used when logging. The default is ratelimit. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP "\fBset log_headers\fR " Enables the ability to log request headers. The default is False. .IP \fBclock_accuracy\fR This should represent how accurate the proxy servers' system clocks are with each other. 1000 means that all the proxies' clock are accurate to each other within 1 millisecond. No ratelimit should be higher than the clock accuracy. The default is 1000. .IP \fBmax_sleep_time_seconds\fR App will immediately return a 498 response if the necessary sleep time ever exceeds the given max_sleep_time_seconds. The default is 60 seconds. .IP \fBlog_sleep_time_seconds\fR To allow visibility into rate limiting set this value > 0 and all sleeps greater than the number will be logged. If set to 0 means disabled. The default is 0. .IP \fBrate_buffer_seconds\fR Number of seconds the rate counter can drop and be allowed to catch up (at a faster than listed rate). A larger number will result in larger spikes in rate but better average accuracy. The default is 5. .IP \fBaccount_ratelimit\fR If set, will limit PUT and DELETE requests to /account_name/container_name. Number is in requests per second. If set to 0 means disabled. The default is 0. .IP \fBaccount_whitelist\fR Comma separated lists of account names that will not be rate limited. The default is ''. .IP \fBaccount_blacklist\fR Comma separated lists of account names that will not be allowed. Returns a 497 response. The default is ''. .IP \fBcontainer_ratelimit_size\fR When set with container_limit_x = r: for containers of size x, limit requests per second to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''. .RE .RS 0 .IP "\fB[filter:domain_remap]\fR" .RE Middleware that translates container and account parts of a domain to path parameters that the proxy server understands. The container.account.storageurl/object gets translated to container.account.storageurl/path_root/account/container/object and account.storageurl/path_root/container/object gets translated to account.storageurl/path_root/account/container/object .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the domain_remap middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#domain_remap\fR. .IP "\fBset log_name\fR" Label used when logging. The default is domain_remap. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP \fBstorage_domain\fR The domain to be used by the middleware. .IP \fBpath_root\fR The path root value for the storage URL. The default is v1. .IP \fBreseller_prefixes\fR Browsers can convert a host header to lowercase, so check that reseller prefix on the account is the correct case. This is done by comparing the items in the reseller_prefixes config option to the found prefix. If they match except for case, the item from reseller_prefixes will be used instead of the found reseller prefix. The reseller_prefixes list is exclusive. If defined, any request with an account prefix not in that list will be ignored by this middleware. Defaults to 'AUTH'. .RE .RS 0 .IP "\fB[filter:catch_errors]\fR" .RE .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the catch_errors middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#catch_errors\fR. .IP "\fBset log_name\fR" Label used when logging. The default is catch_errors. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR " Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .RE .RS 0 .IP "\fB[filter:cname_lookup]\fR" .RE Note: this middleware requires python-dnspython .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the cname_lookup middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#cname_lookup\fR. .IP "\fBset log_name\fR" Label used when logging. The default is cname_lookup. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP \fBstorage_domain\fR The domain to be used by the middleware. .IP \fBlookup_depth\fR How deep in the CNAME chain to look for something that matches the storage domain. The default is 1. .RE .RS 0 .IP "\fB[filter:staticweb]\fR" .RE Note: Put staticweb just after your auth filter(s) in the pipeline .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the staticweb middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#staticweb\fR. .IP \fBcache_timeout\fR Seconds to cache container x-container-meta-web-* header values. The default is 300 seconds. .IP "\fBset log_name\fR" Label used when logging. The default is staticweb. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR " Logging level. The default is INFO. .IP "\fBset log_address\fR " Logging address. The default is /dev/log. .IP "\fBset log_headers\fR" Enables the ability to log request headers. The default is False. .IP "\fBset access_log_name\fR" Label used when logging. The default is staticweb. .IP "\fBset access_log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset access_log_level\fR " Logging level. The default is INFO. .RE .RS 0 .IP "\fB[filter:tempurl]\fR" .RE Note: Put tempurl before slo, dlo, and your auth filter(s) in the pipeline .RS 3 .IP \fBincoming_remove_headers\fR The headers to remove from incoming requests. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. incoming_allow_headers is a list of exceptions to these removals. .IP \fBincoming_allow_headers\fR The headers allowed as exceptions to incoming_remove_headers. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. .IP "\fBoutgoing_remove_headers\fR" The headers to remove from outgoing responses. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. outgoing_allow_headers is a list of exceptions to these removals. .IP "\fBoutgoing_allow_headers\fR" The headers allowed as exceptions to outgoing_remove_headers. Simply a whitespace delimited list of header names and names can optionally end with '*' to indicate a prefix match. .IP "\fBset log_level\fR " .RE .RS 0 .IP "\fB[filter:formpost]\fR" .RE Note: Put formpost just before your auth filter(s) in the pipeline .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the formpost middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#formpost\fR. .RE .RS 0 .IP "\fB[filter:name_check]\fR" .RE Note: Just needs to be placed before the proxy-server in the pipeline. .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the name_check middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#name_check\fR. .IP \fBforbidden_chars\fR Characters that will not be allowed in a name. .IP \fBmaximum_length\fR Maximum number of characters that can be in the name. .IP \fBforbidden_regexp\fR Python regular expressions of substrings that will not be allowed in a name. .RE .RS 0 .IP "\fB[filter:proxy-logging]\fR" .RE Logging for the proxy server now lives in this middleware. If the access_* variables are not set, logging directives from [DEFAULT] without "access_" will be used. .RS 3 .IP \fBuse\fR Entry point for paste.deploy for the proxy_logging middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#proxy_logging\fR. .IP "\fBaccess_log_name\fR" Label used when logging. The default is proxy-server. .IP "\fBaccess_log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBaccess_log_level\fR " Logging level. The default is INFO. .IP \fBaccess_log_address\fR Default is /dev/log. .IP \fBaccess_log_udp_host\fR If set, access_log_udp_host will override access_log_address. Default is unset. .IP \fBaccess_log_udp_port\fR Default is 514. .IP \fBaccess_log_statsd_host\fR You can use log_statsd_* from [DEFAULT], or override them here. Default is localhost. .IP \fBaccess_log_statsd_port\fR Default is 8125. .IP \fBaccess_log_statsd_default_sample_rate\fR Default is 1. .IP \fBaccess_log_statsd_metric_prefix\fR Default is "" (empty-string) .IP \fBaccess_log_headers\fR Default is False. .IP \fBlog_statsd_valid_http_methods\fR What HTTP methods are allowed for StatsD logging (comma-sep); request methods not in this list will have "BAD_METHOD" for the portion of the metric. Default is "GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS". .RE .PD .SH APP SECTION .PD 1 .RS 0 This is indicated by section name [app:proxy-server]. Below are the parameters that are acceptable within this section. .IP \fBuse\fR Entry point for paste.deploy for the proxy server. This is the reference to the installed python egg. This is normally \fBegg:swift#proxy\fR. .IP "\fBset log_name\fR" Label used when logging. The default is proxy-server. .IP "\fBset log_facility\fR" Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR" Logging level. The default is INFO. .IP "\fBset log_address\fR" Logging address. The default is /dev/log. .IP \fBlog_handoffs\fR Log when handoff locations are used. Default is True. .IP \fBrecheck_account_existence\fR Cache timeout in seconds to send memcached for account existence. The default is 60 seconds. .IP \fBrecheck_container_existence\fR Cache timeout in seconds to send memcached for container existence. The default is 60 seconds. .IP \fBobject_chunk_size\fR Chunk size to read from object servers. The default is 8192. .IP \fBclient_chunk_size\fR Chunk size to read from clients. The default is 8192. .IP \fBnode_timeout\fR Request timeout to external services. The default is 10 seconds. .IP \fBclient_timeoutt\fR Timeout to read one chunk from a client. The default is 60 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .IP \fBerror_suppression_interval\fR Time in seconds that must elapse since the last error for a node to be considered no longer error limited. The default is 60 seconds. .IP \fBerror_suppression_limit\fR Error count to consider a node error limited. The default is 10. .IP \fBallow_account_management\fR Whether account PUTs and DELETEs are even callable. If set to 'true' any authorized user may create and delete accounts; if 'false' no one, even authorized, can. The default is false. .IP \fBobject_post_as_copy\fR Set object_post_as_copy = false to turn on fast posts where only the metadata changes are stored as new and the original data file is kept in place. This makes for quicker posts; but since the container metadata isn't updated in this mode, features like container sync won't be able to sync posts. The default is True. .IP \fBaccount_autocreate\fR If set to 'true' authorized accounts that do not yet exist within the Swift cluster will be automatically created. The default is set to false. .IP \fBrate_limit_after_segment\fR Start rate-limiting object segments after the Nth segment of a segmented object. The default is 10 segments. .IP \fBrate_limit_segments_per_sec\fR Once segment rate-limiting kicks in for an object, limit segments served to N per second. The default is 1. .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-proxy-server and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-proxy-server(1) swift-1.13.1/doc/manpages/swift-container-updater.10000664000175400017540000000425612323703611023274 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-updater 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-updater \- Openstack-swift container updater .SH SYNOPSIS .LP .B swift-container-updater [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP The container updater is responsible for updating container information in the account database. It will walk the container path in the system looking for container DBs and sending updates to the account server as needed as it goes along. There are times when account data can not be immediately updated. This usually occurs during failure scenarios or periods of high load. This is where an eventual consistency window will most likely come in to play. In practice, the consistency window is only as large as the frequency at which the updater runs and may not even be noticed as the proxy server will route listing requests to the first account server which responds. The server under load may not be the one that serves subsequent listing requests – one of the other two replicas may handle the listing. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-container-updater and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR container-server.conf(5) swift-1.13.1/doc/manpages/swift-ring-builder.10000664000175400017540000001716612323703611022237 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-ring-builder 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-ring-builder \- Openstack-swift ring builder .SH SYNOPSIS .LP .B swift-ring-builder <...> .SH DESCRIPTION .PP The swift-ring-builder utility is used to create, search and manipulate the swift storage ring. The ring-builder assigns partitions to devices and writes an optimized Python structure to a gzipped, pickled file on disk for shipping out to the servers. The server processes just check the modification time of the file occasionally and reload their in-memory copies of the ring structure as needed. Because of how the ring-builder manages changes to the ring, using a slightly older ring usually just means one of the three replicas for a subset of the partitions will be incorrect, which can be easily worked around. .PP The ring-builder also keeps its own builder file with the ring information and additional data required to build future rings. It is very important to keep multiple backup copies of these builder files. One option is to copy the builder files out to every server while copying the ring files themselves. Another is to upload the builder files into the cluster itself. Complete loss of a builder file will mean creating a new ring from scratch, nearly all partitions will end up assigned to different devices, and therefore nearly all data stored will have to be replicated to new locations. So, recovery from a builder file loss is possible, but data will definitely be unreachable for an extended time. .PP If invoked as 'swift-ring-builder-safe' the directory containing the builder file provided will be locked (via a .lock file in the files parent directory). This provides a basic safe guard against multiple instances of the swift-ring-builder (or other utilities that observe this lock) from attempting to write to or read the builder/ring files while operations are in progress. This can be useful in environments where ring management has been automated but the operator still needs to interact with the rings manually. .SH SEARCH .PD 0 .IP "\fB\fR" .RS 5 .IP "Can be of the form:" .IP "dz-:/_" .IP "Any part is optional, but you must include at least one, examples:" .RS 3 .IP "d74 Matches the device id 74" .IP "z1 Matches devices in zone 1" .IP "z1-1.2.3.4 Matches devices in zone 1 with the ip 1.2.3.4" .IP "1.2.3.4 Matches devices in any zone with the ip 1.2.3.4" .IP "z1:5678 Matches devices in zone 1 using port 5678" .IP ":5678 Matches devices that use port 5678" .IP "/sdb1 Matches devices with the device name sdb1" .IP "_shiny Matches devices with shiny in the meta data" .IP "_'snet: 5.6.7.8' Matches devices with snet: 5.6.7.8 in the meta data" .IP "[::1] Matches devices in any zone with the ip ::1" .IP "z1-[::1]:5678 Matches devices in zone 1 with ip ::1 and port 5678" .RE Most specific example: .RS 3 d74z1-1.2.3.4:5678/sdb1_"snet: 5.6.7.8" .RE Nerd explanation: .RS 3 .IP "All items require their single character prefix except the ip, in which case the - is optional unless the device id or zone is also included." .RE .RE .PD .SH COMMANDS .PD 0 .IP "\fB\fR" .RS 5 Shows information about the ring and the devices within. .RE .IP "\fBsearch\fR " .RS 5 Shows information about matching devices. .RE .IP "\fBadd\fR z-:/_ " .IP "\fBadd\fR rz-:/_ " .IP "\fBadd\fR -r -z -i -p -d -m -w " .RS 5 Adds a device to the ring with the given information. No partitions will be assigned to the new device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. .RE .IP "\fBcreate\fR " .RS 5 Creates with 2^ partitions and . is number of hours to restrict moving a partition more than once. .RE .IP "\fBlist_parts\fR [] .." .RS 5 Returns a 2 column list of all the partitions that are assigned to any of the devices matching the search values given. The first column is the assigned partition number and the second column is the number of device matches for that partition. The list is ordered from most number of matches to least. If there are a lot of devices to match against, this command could take a while to run. .RE .IP "\fBrebalence\fR" .RS 5 Attempts to rebalance the ring by reassigning partitions that haven't been recently reassigned. .RE .IP "\fBremove\fR " .RS 5 Removes the device(s) from the ring. This should normally just be used for a device that has failed. For a device you wish to decommission, it's best to set its weight to 0, wait for it to drain all its data, then use this remove command. This will not take effect until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. .RE .IP "\fBset_info\fR :/_" .RS 5 Resets the device's information. This information isn't used to assign partitions, so you can use 'write_ring' afterward to rewrite the current ring with the newer device information. Any of the parts are optional in the final :/_ parameter; just give what you want to change. For instance set_info d74 _"snet: 5.6.7.8" would just update the meta data for device id 74. .RE .IP "\fBset_min_part_hours\fR " .RS 5 Changes the to the given . This should be set to however long a full replication/update cycle takes. We're working on a way to determine this more easily than scanning logs. .RE .IP "\fBset_weight\fR " .RS 5 Resets the device's weight. No partitions will be reassigned to or from the device until after running 'rebalance'. This is so you can make multiple device changes and rebalance them all just once. .RE .IP "\fBvalidate\fR" .RS 5 Just runs the validation routines on the ring. .RE .IP "\fBwrite_ring\fR" .RS 5 Just rewrites the distributable ring file. This is done automatically after a successful rebalance, so really this is only useful after one or more 'set_info' calls when no rebalance is needed but you want to send out the new device information. .RE \fBQuick list:\fR add create list_parts rebalance remove search set_info set_min_part_hours set_weight validate write_ring \fBExit codes:\fR 0 = ring changed, 1 = ring did not change, 2 = error .PD .SH DOCUMENTATION .LP More in depth documentation about the swift ring and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/overview_ring.html, .BI http://swift.openstack.org/admin_guide.html#managing-the-rings and .BI http://swift.openstack.org swift-1.13.1/doc/manpages/dispersion.conf.50000664000175400017540000000520612323703611021621 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH dispersion.conf 5 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B dispersion.conf \- configuration file for the openstack-swift dispersion tools .SH SYNOPSIS .LP .B dispersion.conf .SH DESCRIPTION .PP This is the configuration file used by the dispersion populate and report tools. The file format consists of the '[dispersion]' module as the header and available parameters. Any line that begins with a '#' symbol is ignored. .SH PARAMETERS .PD 1 .RS 0 .IP "\fBauth_version\fR" Authentication system API version. The default is 1.0. .IP "\fBauth_url\fR" Authentication system URL .IP "\fBauth_user\fR" Authentication system account/user name .IP "\fBauth_key\fR" Authentication system account/user password .IP "\fBswift_dir\fR" Location of openstack-swift configuration and ring files .IP "\fBdispersion_coverage\fR" Percentage of partition coverage to use. The default is 1.0. .IP "\fBretries\fR" Maximum number of attempts .IP "\fBconcurrency\fR" Concurrency to use. The default is 25. .IP "\fBdump_json\fR" Whether to output in json format. The default is no. .IP "\fBcontainer_report\fR" Whether to run the container report. The default is yes. .IP "\fBobject_report\fR" Whether to run the object report. The default is yes. .RE .PD .SH SAMPLE .PD 0 .RS 0 .IP "[dispersion]" .IP "auth_url = https://127.0.0.1:443/auth/v1.0" .IP "auth_user = dpstats:dpstats" .IP "auth_key = dpstats" .IP "swift_dir = /etc/swift" .IP "# keystone_api_insecure = no" .IP "# dispersion_coverage = 1.0" .IP "# retries = 5" .IP "# concurrency = 25" .IP "# dump_json = no" .IP "# container_report = yes" .IP "# object_report = yes" .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-dispersion utilities and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html#cluster-health and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-dispersion-report(1), .BR swift-dispersion-populate(1) swift-1.13.1/doc/manpages/swift-recon.10000664000175400017540000000655012323703611020755 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-recon 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-recon \- Openstack-swift recon middleware cli tool .SH SYNOPSIS .LP .B swift-recon \ [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat] .SH DESCRIPTION .PP The swift-recon cli tool can be used to retrieve various metrics and telemetry information about a cluster that has been collected by the swift-recon middleware. In order to make use of the swift-recon middleware, update the object-server.conf file and enable the recon middleware by adding a pipeline entry and setting its option(s). You can view more information in the example section below. .SH OPTIONS .RS 0 .PD 1 .IP "\fB\fR" account|container|object - Defaults to object server. .IP "\fB-h, --help\fR" show this help message and exit .IP "\fB-v, --verbose\fR" Print verbose information .IP "\fB--suppress\fR" Suppress most connection related errors .IP "\fB-a, --async\fR" Get async stats .IP "\fB--auditor\fR" Get auditor stats .IP "\fB--updater\fR" Get updater stats .IP "\fB--expirer\fR" Get expirer stats .IP "\fB-r, --replication\fR" Get replication stats .IP "\fB-u, --unmounted\fR" Check cluster for unmounted devices .IP "\fB-d, --diskusage\fR" Get disk usage stats .IP "\fB-l, --loadstats\fR" Get cluster load average stats .IP "\fB-q, --quarantined\fR" Get cluster quarantine stats .IP "\fB--md5\fR" Get md5sum of servers ring and compare to local cop .IP "\fB--all\fR" Perform all checks. Equivalent to \-arudlq \-\-md5 .IP "\fB-z ZONE, --zone=ZONE\fR" Only query servers in specified zone .IP "\fB--swiftdir=PATH\fR" Default = /etc/swift .PD .RE .SH EXAMPLE .LP .PD 0 .RS 0 .IP "ubuntu:~$ swift-recon -q --zone 3" .IP "=================================================================" .IP "[2011-10-18 19:36:00] Checking quarantine dirs on 1 hosts... " .IP "[Quarantined objects] low: 4, high: 4, avg: 4, total: 4 " .IP "[Quarantined accounts] low: 0, high: 0, avg: 0, total: 0 " .IP "[Quarantined containers] low: 0, high: 0, avg: 0, total: 0 " .IP "=================================================================" .RE .RS 0 Finally if you also wish to track asynchronous pending’s you will need to setup a cronjob to run the swift-recon-cron script periodically: .IP "*/5 * * * * swift /usr/bin/swift-recon-cron /etc/swift/object-server.conf" .RE .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html Also more specific documentation about swift-recon can be found at .BI http://swift.openstack.org/admin_guide.html#cluster-telemetry-and-monitoring .SH "SEE ALSO" .BR object-server.conf(5), swift-1.13.1/doc/manpages/swift-container-server.10000664000175400017540000000314112323703611023126 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-server 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-server \- Openstack-swift container server .SH SYNOPSIS .LP .B swift-container-server [CONFIG] [-h|--help] [-v|--verbose] .SH DESCRIPTION .PP The Container Server's primary job is to handle listings of objects. It doesn't know where those objects are, just what objects are in a specific container. The listings are stored as sqlite database files, and replicated across the cluster similar to how objects are. Statistics are also tracked that include the total number of objects, and total storage usage for that container. .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-container-server and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html and .BI http://docs.openstack.org .LP .SH "SEE ALSO" .BR container-server.conf(5) swift-1.13.1/doc/manpages/object-server.conf.50000664000175400017540000002212712323703611022215 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH object-server.conf 5 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B object-server.conf \- configuration file for the openstack-swift object server .SH SYNOPSIS .LP .B object-server.conf .SH DESCRIPTION .PP This is the configuration file used by the object server and other object background services, such as; replicator, updater and auditor. The configuration file follows the python-pastedeploy syntax. The file is divided into sections, which are enclosed by square brackets. Each section will contain a certain number of key/value parameters which are described later. Any line that begins with a '#' symbol is ignored. You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION .PD 1 .RS 0 This is indicated by section named [DEFAULT]. Below are the parameters that are acceptable within this section. .IP "\fBbind_ip\fR" IP address the object server should bind to. The default is 0.0.0.0 which will make it bind to all available addresses. .IP "\fBbind_port\fR" TCP port the object server should bind to. The default is 6000. .IP \fBbacklog\fR TCP backlog. Maximum number of allowed pending connections. The default value is 4096. .IP \fBworkers\fR The number of pre-forked processes that will accept connections. Zero means no fork. The default is auto which will make the server try to match the number of effective cpu cores if python multiprocessing is available (included with most python distributions >= 2.6) or fallback to one. It's worth noting that individual workers will use many eventlet co-routines to service multiple concurrent requests. .IP \fBmax_clients\fR Maximum number of clients one worker can process simultaneously (it will actually accept(2) N + 1). Setting this to one (1) will only handle one request at a time, without accepting another request concurrently. The default is 1024. .IP \fBuser\fR The system user that the object server will run as. The default is swift. .IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. .IP \fBdevices\fR Parent directory or where devices are mounted. Default is /srv/node. .IP \fBmount_check\fR Whether or not check if the devices are mounted to prevent accidentally writing to the root device. The default is set to true. .IP \fBlog_name\fR Label used when logging. The default is swift. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .RE .PD .SH PIPELINE SECTION .PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that are acceptable within this section. .IP "\fBpipeline\fR" It is used when you need to apply a number of filters. It is a list of filters ended by an application. The normal pipeline is "healthcheck recon object-server". .RE .PD .SH APP SECTION .PD 1 .RS 0 This is indicated by section name [app:object-server]. Below are the parameters that are acceptable within this section. .IP "\fBuse\fR" Entry point for paste.deploy for the object server. This is the reference to the installed python egg. This is normally \fBegg:swift#object\fR. .IP "\fBset log_name\fR Label used when logging. The default is object-server. .IP "\fBset log_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP "\fB set log_level\fR Logging level. The default is INFO. .IP "\fB set log_requests\fR Enables request logging. The default is True. .IP "\fB set log_address\fR Logging address. The default is /dev/log. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .RE .PD .SH FILTER SECTION .PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. Below are the filters available and respective acceptable parameters. .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#healthcheck\fR. .IP "\fBdisable_path\fR" An optional filesystem path which, if present, will cause the healthcheck URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE". .RE .RS 0 .IP "\fB[filter:recon]\fR" .RE .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the recon middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#recon\fR. .IP "\fBrecon_cache_path\fR" The recon_cache_path simply sets the directory where stats for a few items will be stored. Depending on the method of deployment you may need to create this directory manually and ensure that swift has read/write. The default is /var/cache/swift. .RE .PD .SH ADDITIONAL SECTIONS .PD 1 .RS 0 The following sections are used by other swift-object services, such as replicator, updater, auditor. .IP "\fB[object-replicator]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is object-replicator. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBdaemonize\fR Whether or not to run replication as a daemon. The default is yes. .IP \fBrun_pause\fR Time in seconds to wait between replication passes. The default is 30. .IP \fBconcurrency\fR Number of replication workers to spawn. The default is 1. .IP \fBstats_interval\fR Interval in seconds between logging replication statistics. The default is 300. .IP \fBrsync_timeout\fR Max duration of a partition rsync. The default is 900 seconds. .IP \fBrsync_io_timeout\fR Passed to rsync for I/O OP timeout. The default is 30 seconds. .IP \fBrsync_bwlimit\fR Passed to rsync for bandwidth limit in kB/s. The default is 0 (unlimited). .IP \fBhttp_timeout\fR Max duration of an HTTP request. The default is 60 seconds. .IP \fBlockup_timeout\fR Attempts to kill all workers if nothing replicates for lockup_timeout seconds. The default is 1800 seconds. .IP \fBreclaim_age\fR Time elapsed in seconds before an object can be reclaimed. The default is 604800 seconds. .IP \fBrecon_enable\fR Enable logging of replication stats for recon. The default is on. .IP "\fBrecon_cache_path\fR" The recon_cache_path simply sets the directory where stats for a few items will be stored. Depending on the method of deployment you may need to create this directory manually and ensure that swift has read/write.The default is /var/cache/swift. .RE .RS 0 .IP "\fB[object-updater]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is object-updater. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBinterval\fR Minimum time for a pass to take. The default is 300 seconds. .IP \fBconcurrency\fR Number of reaper workers to spawn. The default is 1. .IP \fBnode_timeout\fR Request timeout to external services. The default is 10 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .IP \fBslowdown\fR Slowdown will sleep that amount between objects. The default is 0.01 seconds. .RE .PD .RS 0 .IP "\fB[object-auditor]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is object-auditor. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBfiles_per_second\fR Maximum files audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 20. .IP \fBbytes_per_second\fR Maximum bytes audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 10000000. .IP \fBlog_time\fR The default is 3600 seconds. .IP \fBzero_byte_files_per_second\fR The default is 50. .RE .SH DOCUMENTATION .LP More in depth documentation about the swift-object-server and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-object-server(1), swift-1.13.1/doc/manpages/swift-account-reaper.10000664000175400017540000000360212323703611022552 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-account-reaper 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-account-reaper \- Openstack-swift account reaper .SH SYNOPSIS .LP .B swift-account-reaper [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP Removes data from status=DELETED accounts. These are accounts that have been asked to be removed by the reseller via services remove_storage_account XMLRPC call. .PP The account is not deleted immediately by the services call, but instead the account is simply marked for deletion by setting the status column in the account_stat table of the account database. This account reaper scans for such accounts and removes the data in the background. The background deletion process will occur on the primary account server for the account. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-auditor and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR account-server.conf(5) swift-1.13.1/doc/manpages/swift-account-auditor.10000664000175400017540000000315012323703611022741 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-account-auditor 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-account-auditor \- Openstack-swift account auditor .SH SYNOPSIS .LP .B swift-account-auditor [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP The account auditor crawls the local account system checking the integrity of accounts objects. If corruption is found (in the case of bit rot, for example), the file is quarantined, and replication will replace the bad file from another replica. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-account-auditor and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR account-server.conf(5) swift-1.13.1/doc/manpages/swift-object-expirer.10000664000175400017540000000410212323703611022560 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-expirer 1 "3/15/2012" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-expirer \- Openstack-swift object expirer .SH SYNOPSIS .LP .B swift-object-expirer [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP The swift-object-expirer offers scheduled deletion of objects. The Swift client would use the X-Delete-At or X-Delete-After headers during an object PUT or POST and the cluster would automatically quit serving that object at the specified time and would shortly thereafter remove the object from the system. The X-Delete-At header takes a Unix Epoch timestamp, in integer form; for example: 1317070737 represents Mon Sep 26 20:58:57 2011 UTC. The X-Delete-After header takes a integer number of seconds. The proxy server that receives the request will convert this header into an X-Delete-At header using its current time plus the value given. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-expirer can be foud at .BI http://swift.openstack.org/overview_expiring_objects.html and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR object-expirer.conf(5) swift-1.13.1/doc/manpages/swift-container-auditor.10000664000175400017540000000320112323703611023264 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-auditor 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-auditor \- Openstack-swift container auditor .SH SYNOPSIS .LP .B swift-container-auditor [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP The container auditor crawls the local container system checking the integrity of container objects. If corruption is found (in the case of bit rot, for example), the file is quarantined, and replication will replace the bad file from another replica. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-container-auditor and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR container-server.conf(5) swift-1.13.1/doc/manpages/swift-object-replicator.10000664000175400017540000000411712323703611023254 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-object-replicator 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-object-replicator \- Openstack-swift object replicator .SH SYNOPSIS .LP .B swift-object-replicator [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP Replication is designed to keep the system in a consistent state in the face of temporary error conditions like network outages or drive failures. The replication processes compare local data with each remote copy to ensure they all contain the latest version. Object replication uses a hash list to quickly compare subsections of each partition. .PP Replication updates are push based. For object replication, updating is just a matter of rsyncing files to the peer. The replicator also ensures that data is removed from the system. When an object item is deleted a tombstone is set as the latest version of the item. The replicator will see the tombstone and ensure that the item is removed from the entire system. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-object-replicator and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR object-server.conf(5) swift-1.13.1/doc/manpages/swift-account-replicator.10000664000175400017540000000420112323703611023434 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-account-replicator 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-account-replicator \- Openstack-swift account replicator .SH SYNOPSIS .LP .B swift-account-replicator [CONFIG] [-h|--help] [-v|--verbose] [-o|--once] .SH DESCRIPTION .PP Replication is designed to keep the system in a consistent state in the face of temporary error conditions like network outages or drive failures. The replication processes compare local data with each remote copy to ensure they all contain the latest version. Account replication uses a combination of hashes and shared high water marks to quickly compare subsections of each partition. .PP Replication updates are push based. Account replication push missing records over HTTP or rsync whole database files. The replicator also ensures that data is removed from the system. When an account item is deleted a tombstone is set as the latest version of the item. The replicator will see the tombstone and ensure that the item is removed from the entire system. The options are as follows: .RS 4 .PD 0 .IP "-v" .IP "--verbose" .RS 4 .IP "log to console" .RE .IP "-o" .IP "--once" .RS 4 .IP "only run one pass of daemon" .RE .PD .RE .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-account-replicator and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR account-server.conf(5) swift-1.13.1/doc/manpages/swift-dispersion-report.10000664000175400017540000000772612323703611023345 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-dispersion-report 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-dispersion-report \- Openstack-swift dispersion report .SH SYNOPSIS .LP .B swift-dispersion-report [-d|--debug] [-j|--dump-json] [-p|--partitions] [--container-only|--object-only] [--insecure] [conf_file] .SH DESCRIPTION .PP This is one of the swift-dispersion utilities that is used to evaluate the overall cluster health. This is accomplished by checking if a set of deliberately distributed containers and objects are currently in their proper places within the cluster. .PP For instance, a common deployment has three replicas of each object. The health of that object can be measured by checking if each replica is in its proper place. If only 2 of the 3 is in place the object's health can be said to be at 66.66%, where 100% would be perfect. .PP Once the \fBswift-dispersion-populate\fR has been used to populate the dispersion account, one should run the \fBswift-dispersion-report\fR tool repeatedly for the life of the cluster, in order to check the health of each of these containers and objects. .PP These tools need direct access to the entire cluster and to the ring files. Installing them on a proxy server will probably do or a box used for swift administration purposes that also contains the common swift packages and ring. Both \fBswift-dispersion-populate\fR and \fBswift-dispersion-report\fR use the same configuration file, /etc/swift/dispersion.conf . The account used by these tool should be a dedicated account for the dispersion stats and also have admin privileges. .SH OPTIONS .RS 0 .PD 1 .IP "\fB-d, --debug\fR" output any 404 responses to standard error .SH OPTIONS .RS 0 .PD 1 .IP "\fB-j, --dump-json\fR" output dispersion report in json format .SH OPTIONS .RS 0 .PD 1 .IP "\fB-p, --partitions\fR" output the partition numbers that have any missing replicas .SH OPTIONS .RS 0 .PD 1 .IP "\fB--container-only\fR" Only run the container report .SH OPTIONS .RS 0 .PD 1 .IP "\fB--object-only\fR" Only run the object report .SH OPTIONS .RS 0 .PD 1 .IP "\fB--insecure\fR" Allow accessing insecure keystone server. The keystone's certificate will not be verified. .SH CONFIGURATION .PD 0 Example \fI/etc/swift/dispersion.conf\fR: .RS 3 .IP "[dispersion]" .IP "auth_url = https://127.0.0.1:443/auth/v1.0" .IP "auth_user = dpstats:dpstats" .IP "auth_key = dpstats" .IP "swift_dir = /etc/swift" .IP "# dispersion_coverage = 1.0" .IP "# retries = 5" .IP "# concurrency = 25" .IP "# dump_json = no" .IP "# endpoint_type = publicURL" .RE .PD .SH EXAMPLE .PP .PD 0 $ swift-dispersion-report .RS 1 .IP "Queried 2622 containers for dispersion reporting, 31s, 0 retries" .IP "100.00% of container copies found (7866 of 7866)" .IP "Sample represents 1.00% of the container partition space" .IP "Queried 2621 objects for dispersion reporting, 22s, 0 retries" .IP "100.00% of object copies found (7863 of 7863)" .IP "Sample represents 1.00% of the object partition space" .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-dispersion utilities and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html#cluster-health and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-dispersion-populate(1), .BR dispersion.conf (5) swift-1.13.1/doc/manpages/container-server.conf.50000664000175400017540000002247512323703611022737 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2012 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH container-server.conf 5 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B container-server.conf \- configuration file for the openstack-swift container server .SH SYNOPSIS .LP .B container-server.conf .SH DESCRIPTION .PP This is the configuration file used by the container server and other container background services, such as; replicator, updater, auditor and sync. The configuration file follows the python-pastedeploy syntax. The file is divided into sections, which are enclosed by square brackets. Each section will contain a certain number of key/value parameters which are described later. Any line that begins with a '#' symbol is ignored. You can find more information about python-pastedeploy configuration format at \fIhttp://pythonpaste.org/deploy/#config-format\fR .SH GLOBAL SECTION .PD 1 .RS 0 This is indicated by section named [DEFAULT]. Below are the parameters that are acceptable within this section. .IP "\fBbind_ip\fR" IP address the container server should bind to. The default is 0.0.0.0 which will make it bind to all available addresses. .IP "\fBbind_port\fR" TCP port the container server should bind to. The default is 6001. .IP \fBbacklog\fR TCP backlog. Maximum number of allowed pending connections. The default value is 4096. .IP \fBworkers\fR The number of pre-forked processes that will accept connections. Zero means no fork. The default is auto which will make the server try to match the number of effective cpu cores if python multiprocessing is available (included with most python distributions >= 2.6) or fallback to one. It's worth noting that individual workers will use many eventlet co-routines to service multiple concurrent requests. .IP \fBmax_clients\fR Maximum number of clients one worker can process simultaneously (it will actually accept(2) N + 1). Setting this to one (1) will only handle one request at a time, without accepting another request concurrently. The default is 1024. .IP \fBuser\fR The system user that the container server will run as. The default is swift. .IP \fBswift_dir\fR Swift configuration directory. The default is /etc/swift. .IP \fBdevices\fR Parent directory or where devices are mounted. Default is /srv/node. .IP \fBmount_check\fR Whether or not check if the devices are mounted to prevent accidentally writing to the root device. The default is set to true. .IP \fBlog_name\fR Label used when logging. The default is swift. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .RE .PD .SH PIPELINE SECTION .PD 1 .RS 0 This is indicated by section name [pipeline:main]. Below are the parameters that are acceptable within this section. .IP "\fBpipeline\fR" It is used when you need to apply a number of filters. It is a list of filters ended by an application. The normal pipeline is "healthcheck recon container-server". .RE .PD .SH APP SECTION .PD 1 .RS 0 This is indicated by section name [app:container-server]. Below are the parameters that are acceptable within this section. .IP "\fBuse\fR" Entry point for paste.deploy for the container server. This is the reference to the installed python egg. This is normally \fBegg:swift#container\fR. .IP "\fBset log_name\fR Label used when logging. The default is container-server. .IP "\fBset log_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP "\fBset log_level\fR Logging level. The default is INFO. .IP "\fBset log_requests\fR Enables request logging. The default is True. .IP "\fBset log_address\fR Logging address. The default is /dev/log. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .RE .PD .SH FILTER SECTION .PD 1 .RS 0 Any section that has its name prefixed by "filter:" indicates a filter section. Filters are used to specify configuration parameters for specific swift middlewares. Below are the filters available and respective acceptable parameters. .IP "\fB[filter:healthcheck]\fR" .RE .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the healthcheck middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#healthcheck\fR. .IP "\fBdisable_path\fR" An optional filesystem path which, if present, will cause the healthcheck URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE". .RE .RS 0 .IP "\fB[filter:recon]\fR" .RS 3 .IP "\fBuse\fR" Entry point for paste.deploy for the recon middleware. This is the reference to the installed python egg. This is normally \fBegg:swift#recon\fR. .IP "\fBrecon_cache_path\fR" The recon_cache_path simply sets the directory where stats for a few items will be stored. Depending on the method of deployment you may need to create this directory manually and ensure that swift has read/write. The default is /var/cache/swift. .RE .PD .SH ADDITIONAL SECTIONS .PD 1 .RS 0 The following sections are used by other swift-container services, such as replicator, updater, auditor and sync. .IP "\fB[container-replicator]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is container-replicator. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBvm_test_mode\fR Indicates that you are using a VM environment. The default is no. .IP \fBer_diff\fR The default is 1000. .IP \fBmax_diffs\fR This caps how long the replicator will spend trying to sync a given database per pass so the other databases don't get starved. The default is 100. .IP \fBconcurrency\fR Number of replication workers to spawn. The default is 8. .IP "\fBrun_pause [deprecated]\fR" Time in seconds to wait between replication passes. The default is 10. .IP \fBinterval\fR Replaces run_pause with the more standard "interval", which means the replicator won't pause unless it takes less than the interval set. The default is 30. .IP \fBnode_timeout\fR Request timeout to external services. The default is 10 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .IP \fBreclaim_age\fR Time elapsed in seconds before an container can be reclaimed. The default is 604800 seconds. .RE .RS 0 .IP "\fB[container-updater]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is container-updater. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBinterval\fR Minimum time for a pass to take. The default is 300 seconds. .IP \fBconcurrency\fR Number of reaper workers to spawn. The default is 4. .IP \fBnode_timeout\fR Request timeout to external services. The default is 3 seconds. .IP \fBconn_timeout\fR Connection timeout to external services. The default is 0.5 seconds. .IP \fBslowdown\fR Slowdown will sleep that amount between containers. The default is 0.01 seconds. .IP \fBaccount_suppression_time\fR Seconds to suppress updating an account that has generated an error. The default is 60 seconds. .RE .PD .RS 0 .IP "\fB[container-auditor]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is container-auditor. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBinterval\fR Will audit, at most, 1 container per device per interval. The default is 1800 seconds. .IP \fBcontainers_per_second\fR Maximum containers audited per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 200. .RE .RS 0 .IP "\fB[container-sync]\fR" .RE .RS 3 .IP \fBlog_name\fR Label used when logging. The default is container-sync. .IP \fBlog_facility\fR Syslog log facility. The default is LOG_LOCAL0. .IP \fBlog_level\fR Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. .IP \fBsync_proxy\fR If you need to use an HTTP Proxy, set it here; defaults to no proxy. .IP \fBinterval\fR Will audit, at most, each container once per interval. The default is 300 seconds. .IP \fBcontainer_time\fR Maximum amount of time to spend syncing each container per pass. The default is 60 seconds. .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-container-server and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-container-server(1) swift-1.13.1/doc/manpages/swift-proxy-server.10000664000175400017540000000344212323703611022331 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-proxy-server 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-proxy-server \- Openstack-swift proxy server. .SH SYNOPSIS .LP .B swift-proxy-server [CONFIG] [-h|--help] [-v|--verbose] .SH DESCRIPTION .PP The Swift Proxy Server is responsible for tying together the rest of the Swift architecture. For each request, it will look up the location of the account, container, or object in the ring and route the request accordingly. The public API is also exposed through the Proxy Server. A large number of failures are also handled in the Proxy Server. For example, if a server is unavailable for an object PUT, it will ask the ring for a handoff server and route there instead. When objects are streamed to or from an object server, they are streamed directly through the proxy server to or from the user the proxy server does not spool them. .SH DOCUMENTATION .LP More in depth documentation in regards to .BI swift-proxy-server and also about Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR proxy-server.conf(5) swift-1.13.1/doc/manpages/swift-dispersion-populate.10000664000175400017540000000750512323703611023656 0ustar jenkinsjenkins00000000000000.\" .\" Author: Joao Marcelo Martins or .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-dispersion-populate 1 "8/26/2011" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-dispersion-populate \- Openstack-swift dispersion populate .SH SYNOPSIS .LP .B swift-dispersion-populate [--container-suffix-start] [--object-suffix-start] [--container-only|--object-only] [--insecure] [conf_file] .SH DESCRIPTION .PP This is one of the swift-dispersion utilities that is used to evaluate the overall cluster health. This is accomplished by checking if a set of deliberately distributed containers and objects are currently in their proper places within the cluster. .PP For instance, a common deployment has three replicas of each object. The health of that object can be measured by checking if each replica is in its proper place. If only 2 of the 3 is in place the object's health can be said to be at 66.66%, where 100% would be perfect. .PP We need to place the containers and objects throughout the system so that they are on distinct partitions. The \fBswift-dispersion-populate\fR tool does this by making up random container and object names until they fall on distinct partitions. Last, and repeatedly for the life of the cluster, we need to run the \fBswift-dispersion-report\fR tool to check the health of each of these containers and objects. .PP These tools need direct access to the entire cluster and to the ring files. Installing them on a proxy server will probably do or a box used for swift administration purposes that also contains the common swift packages and ring. Both \fBswift-dispersion-populate\fR and \fBswift-dispersion-report\fR use the same configuration file, /etc/swift/dispersion.conf . The account used by these tool should be a dedicated account for the dispersion stats and also have admin privileges. .SH OPTIONS .RS 0 .PD 1 .IP "\fB--insecure\fR" Allow accessing insecure keystone server. The keystone's certificate will not be verified. .IP "\fB--container-suffix-start=NUMBER\fR" Start container suffix at NUMBER and resume population at this point; default: 0 .IP "\fB--object-suffix-start=NUMBER\fR" Start object suffix at NUMBER and resume population at this point; default: 0 .IP "\fB--object-only\fR" Only run object population .IP "\fB--container-only\fR" Only run container population .IP "\fB--object-only\fR" Only run object population .SH CONFIGURATION .PD 0 Example \fI/etc/swift/dispersion.conf\fR: .RS 3 .IP "[dispersion]" .IP "auth_url = https://127.0.0.1:443/auth/v1.0" .IP "auth_user = dpstats:dpstats" .IP "auth_key = dpstats" .IP "swift_dir = /etc/swift" .IP "# dispersion_coverage = 1.0" .IP "# retries = 5" .IP "# concurrency = 25" .IP "# endpoint_type = publicURL" .RE .PD .SH EXAMPLE .PP .PD 0 $ swift-dispersion-populate .RS 1 .IP "Created 2621 containers for dispersion reporting, 38s, 0 retries" .IP "Created 2621 objects for dispersion reporting, 27s, 0 retries" .RE .PD .SH DOCUMENTATION .LP More in depth documentation about the swift-dispersion utilities and also Openstack-Swift as a whole can be found at .BI http://swift.openstack.org/admin_guide.html#cluster-health and .BI http://swift.openstack.org .SH "SEE ALSO" .BR swift-dispersion-report(1), .BR dispersion.conf (5) swift-1.13.1/doc/manpages/swift-container-info.10000664000175400017540000000355412323703611022563 0ustar jenkinsjenkins00000000000000.\" .\" Author: Madhuri Kumari .\" Copyright (c) 2010-2011 OpenStack Foundation. .\" .\" Licensed under the Apache License, Version 2.0 (the "License"); .\" you may not use this file except in compliance with the License. .\" You may obtain a copy of the License at .\" .\" http://www.apache.org/licenses/LICENSE-2.0 .\" .\" Unless required by applicable law or agreed to in writing, software .\" distributed under the License is distributed on an "AS IS" BASIS, .\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or .\" implied. .\" See the License for the specific language governing permissions and .\" limitations under the License. .\" .TH swift-container-info 1 "3/20/2013" "Linux" "OpenStack Swift" .SH NAME .LP .B swift-container-info \- Openstack-swift container-info tool .SH SYNOPSIS .LP .B swift-container-info [CONTAINER_DB_FILE] [SWIFT_DIR] .SH DESCRIPTION .PP This is a very simple swift tool that allows a swiftop engineer to retrieve information about a container that is located on the storage node. One calls the tool with a given container db file as it is stored on the storage node system. It will then return several information about that container such as; .PD 0 .IP "- Account it belongs to" .IP "- Container " .IP "- Created timestamp " .IP "- Put timestamp " .IP "- Delete timestamp " .IP "- Object count " .IP "- Bytes used " .IP "- Reported put timestamp " .IP "- Reported delete timestamp " .IP "- Reported object count " .IP "- Reported bytes used " .IP "- Hash " .IP "- ID " .IP "- User metadata " .IP "- X-Container-Sync-Point 1 " .IP "- X-Container-Sync-Point 2 " .IP "- Location on the ring " .PD .SH DOCUMENTATION .LP More documentation about Openstack-Swift can be found at .BI http://swift.openstack.org/index.html .SH "SEE ALSO" .BR swift-get-nodes(1), .BR swift-object-info(1) swift-1.13.1/doc/source/0000775000175400017540000000000012323703665016143 5ustar jenkinsjenkins00000000000000swift-1.13.1/doc/source/development_guidelines.rst0000664000175400017540000000733012323703611023421 0ustar jenkinsjenkins00000000000000====================== Development Guidelines ====================== ----------------- Coding Guidelines ----------------- For the most part we try to follow PEP 8 guidelines which can be viewed here: http://www.python.org/dev/peps/pep-0008/ There is a useful pep8 command line tool for checking files for pep8 compliance which can be installed with ``easy_install pep8``. ------------------ Testing Guidelines ------------------ Swift has a comprehensive suite of tests that are run on all submitted code, and it is recommended that developers execute the tests themselves to catch regressions early. Developers are also expected to keep the test suite up-to-date with any submitted code changes. Swift's suite of unit tests can be executed in an isolated environment with Tox: http://tox.testrun.org/ To execute the unit tests: * Install Tox: - `pip install tox` * If you do not have python 2.6 installed (as in 12.04): - Add `export TOXENV=py27,pep8` to your `~/.bashrc` - `. ~/.bashrc` * Run Tox from the root of the swift repo: - `tox` Remarks: If you installed using: `cd ~/swift; sudo python setup.py develop`, you may need to do: `cd ~/swift; sudo chown -R swift:swift swift.egg-info` prior to running tox. If you ever encounter DistributionNotFound, try to use `tox --recreate` or removing .tox directory to force tox to recreate the dependency list * Optionally, run only specific tox builds: - `tox -e pep8,py26` ------------ Coding Style ------------ Swift use flake8 with the OpenStack `hacking`_ module to enforce coding style. Install flake8 and hacking with pip or by the packages of your Operating System. It is advised to integrate flake8+hacking with your editor to get it automated and not get `caught` by Jenkins. For example for Vim the `syntastic`_ plugin can do this for you. .. _`hacking`: https://pypi.python.org/pypi/hacking .. _`syntastic`: https://github.com/scrooloose/syntastic ------------------------ Documentation Guidelines ------------------------ The documentation in docstrings should follow the PEP 257 conventions (as mentioned in the PEP 8 guidelines). More specifically: 1. Triple qutes should be used for all docstrings. 2. If the docstring is simple and fits on one line, then just use one line. 3. For docstrings that take multiple lines, there should be a newline after the opening quotes, and before the closing quotes. 4. Sphinx is used to build documentation, so use the restructured text markup to designate parameters, return values, etc. Documentation on the sphinx specific markup can be found here: http://sphinx.pocoo.org/markup/index.html Installing Sphinx: #. Install sphinx (On Ubuntu: `sudo apt-get install python-sphinx`) #. `python setup.py build_sphinx` --------------------- License and Copyright --------------------- You can have the following copyright and license statement at the top of each source file. Copyright assignment is optional. New files should contain the current year. Substantial updates can have another year added, and date ranges are not needed.:: # Copyright (c) 2013 OpenStack Foundation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. swift-1.13.1/doc/source/development_saio.rst0000664000175400017540000004150012323703614022224 0ustar jenkinsjenkins00000000000000======================= SAIO - Swift All In One ======================= --------------------------------------------- Instructions for setting up a development VM --------------------------------------------- This section documents setting up a virtual machine for doing Swift development. The virtual machine will emulate running a four node Swift cluster. * Get an Ubuntu 12.04 LTS (Precise Pangolin) server image or try something Fedora/CentOS. * Create guest virtual machine from the image. Additional information about setting up a Swift development snapshot on other distributions is available on the wiki at http://wiki.openstack.org/SAIOInstructions. ---------------------------- What's in a ---------------------------- Much of the configuration described in this guide requires escalated administrator (``root``) privileges; however, we assume that administrator logs in as an unprivileged user and can use ``sudo`` to run privileged commands. Swift processes also run under a separate user and group, set by configuration option, and referenced as ``:``. The default user is ``swift``, which may not exist on your system. These instructions are intended to allow a developer to use his/her username for ``:``. ----------------------- Installing dependencies ----------------------- * On ``apt`` based systems:: sudo apt-get update sudo apt-get install curl gcc memcached rsync sqlite3 xfsprogs \ git-core libffi-dev python-setuptools sudo apt-get install python-coverage python-dev python-nose \ python-simplejson python-xattr python-eventlet \ python-greenlet python-pastedeploy \ python-netifaces python-pip python-dnspython \ python-mock * On ``yum`` based systems:: sudo yum update sudo yum install curl gcc memcached rsync sqlite xfsprogs git-core \ libffi-devel xinetd python-setuptools \ python-coverage python-devel python-nose \ python-simplejson pyxattr python-eventlet \ python-greenlet python-paste-deploy \ python-netifaces python-pip python-dns \ python-mock This installs necessary system dependencies; and *most* of the python dependencies. Later in the process setuptools/distribute or pip will install and/or upgrade some other stuff - it's getting harder to avoid. You can also install anything else you want, like screen, ssh, vim, etc. Next, choose either :ref:`partition-section` or :ref:`loopback-section`. .. _partition-section: Using a partition for storage ============================= If you are going to use a separate partition for Swift data, be sure to add another device when creating the VM, and follow these instructions: #. Set up a single partition:: sudo fdisk /dev/sdb sudo mkfs.xfs /dev/sdb1 #. Edit ``/etc/fstab`` and add:: /dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0 #. Create the mount point and the individualized links:: sudo mkdir /mnt/sdb1 sudo mount /mnt/sdb1 sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 sudo chown ${USER}:${USER} /mnt/sdb1/* sudo mkdir /srv for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done sudo mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 \ /srv/4/node/sdb4 /var/run/swift sudo chown -R ${USER}:${USER} /var/run/swift # **Make sure to include the trailing slash after /srv/$x/** for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done #. Next, skip to :ref:`common-dev-section`. .. _loopback-section: Using a loopback device for storage =================================== If you want to use a loopback device instead of another partition, follow these instructions: #. Create the file for the loopback device:: sudo mkdir /srv sudo truncate -s 1GB /srv/swift-disk sudo mkfs.xfs /srv/swift-disk Modify size specified in the ``truncate`` command to make a larger or smaller partition as needed. #. Edit `/etc/fstab` and add:: /srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0 #. Create the mount point and the individualized links:: sudo mkdir /mnt/sdb1 sudo mount /mnt/sdb1 sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 sudo chown ${USER}:${USER} /mnt/sdb1/* for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done sudo mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift sudo chown -R ${USER}:${USER} /var/run/swift # **Make sure to include the trailing slash after /srv/$x/** for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done .. _common-dev-section: Common Post-Device Setup ======================== Add the following lines to ``/etc/rc.local`` (before the ``exit 0``):: mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4 chown : /var/cache/swift* mkdir -p /var/run/swift chown : /var/run/swift Note that on some systems you might have to create ``/etc/rc.local``. On Fedora 19 or later, you need to place these in ``/etc/rc.d/rc.local``. ---------------- Getting the code ---------------- #. Check out the python-swiftclient repo:: cd $HOME; git clone https://github.com/openstack/python-swiftclient.git #. Build a development installation of python-swiftclient:: cd $HOME/python-swiftclient; sudo python setup.py develop; cd - #. Check out the swift repo:: git clone https://github.com/openstack/swift.git #. Build a development installation of swift:: cd $HOME/swift; sudo python setup.py develop; cd - Fedora 19 or later users might have to perform the following if development installation of swift fails:: sudo pip install -U xattr #. Install swift's test dependencies:: sudo pip install -r swift/test-requirements.txt ---------------- Setting up rsync ---------------- #. Create ``/etc/rsyncd.conf``:: sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/ sudo sed -i "s//${USER}/" /etc/rsyncd.conf Here is the default ``rsyncd.conf`` file contents maintained in the repo that is copied and fixed up above: .. literalinclude:: /../saio/rsyncd.conf #. On Ubuntu, edit the following line in ``/etc/default/rsync``:: RSYNC_ENABLE=true On Fedora, edit the following line in ``/etc/xinetd.d/rsync``:: disable = no One might have to create the above files to perform the edits. #. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``:: sudo setenforce Permissive Or just allow rsync full access:: sudo setsebool -P rsync_full_access 1 #. Start the rsync daemon * On Ubuntu, run:: sudo service rsync restart * On Fedora, run:: sudo systemctl restart xinetd.service sudo systemctl enable rsyncd.service sudo systemctl start rsyncd.service * On other xinetd based systems simply run:: sudo service xinetd restart #. Verify rsync is accepting connections for all servers:: rsync rsync://pub@localhost/ You should see the following output from the above command:: account6012 account6022 account6032 account6042 container6011 container6021 container6031 container6041 object6010 object6020 object6030 object6040 ------------------ Starting memcached ------------------ On non-Ubuntu distros you need to ensure memcached is running:: sudo service memcached start sudo chkconfig memcached on or:: sudo systemctl enable memcached.service sudo systemctl start memcached.service The tempauth middleware stores tokens in memcached. If memcached is not running, tokens cannot be validated, and accessing Swift becomes impossible. --------------------------------------------------- Optional: Setting up rsyslog for individual logging --------------------------------------------------- #. Install the swift rsyslogd configuration:: sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/ Be sure to review that conf file to determine if you want all the logs in one file vs. all the logs separated out, and if you want hourly logs for stats processing. For convenience, we provide its default contents below: .. literalinclude:: /../saio/rsyslog.d/10-swift.conf #. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the "GLOBAL DIRECTIVES" section):: $PrivDropToGroup adm #. If using hourly logs (see above) perform:: sudo mkdir -p /var/log/swift/hourly Otherwise perform:: sudo mkdir -p /var/log/swift #. Setup the logging directory and start syslog: * On Ubuntu:: sudo chown -R syslog.adm /var/log/swift sudo chmod -R g+w /var/log/swift sudo service rsyslog restart * On Fedora:: sudo chown -R root:adm /var/log/swift sudo chmod -R g+w /var/log/swift sudo systemctl restart rsyslog.service --------------------- Configuring each node --------------------- After performing the following steps, be sure to verify that Swift has access to resulting configuration files (sample configuration files are provided with all defaults in line-by-line comments). #. Optionally remove an existing swift directory:: sudo rm -rf /etc/swift #. Populate the ``/etc/swift`` directory itself:: cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd - sudo chown -R ${USER}:${USER} /etc/swift #. Update ```` references in the Swift config files:: find /etc/swift/ -name \*.conf | xargs sudo sed -i "s//${USER}/" The contents of the configuration files provided by executing the above commands are as follows: #. ``/etc/swift/swift.conf`` .. literalinclude:: /../saio/swift/swift.conf #. ``/etc/swift/proxy-server.conf`` .. literalinclude:: /../saio/swift/proxy-server.conf #. ``/etc/swift/object-expirer.conf`` .. literalinclude:: /../saio/swift/object-expirer.conf #. ``/etc/swift/account-server/1.conf`` .. literalinclude:: /../saio/swift/account-server/1.conf #. ``/etc/swift/container-server/1.conf`` .. literalinclude:: /../saio/swift/container-server/1.conf #. ``/etc/swift/object-server/1.conf`` .. literalinclude:: /../saio/swift/object-server/1.conf #. ``/etc/swift/account-server/2.conf`` .. literalinclude:: /../saio/swift/account-server/2.conf #. ``/etc/swift/container-server/2.conf`` .. literalinclude:: /../saio/swift/container-server/2.conf #. ``/etc/swift/object-server/2.conf`` .. literalinclude:: /../saio/swift/object-server/2.conf #. ``/etc/swift/account-server/3.conf`` .. literalinclude:: /../saio/swift/account-server/3.conf #. ``/etc/swift/container-server/3.conf`` .. literalinclude:: /../saio/swift/container-server/3.conf #. ``/etc/swift/object-server/3.conf`` .. literalinclude:: /../saio/swift/object-server/3.conf #. ``/etc/swift/account-server/4.conf`` .. literalinclude:: /../saio/swift/account-server/4.conf #. ``/etc/swift/container-server/4.conf`` .. literalinclude:: /../saio/swift/container-server/4.conf #. ``/etc/swift/object-server/4.conf`` .. literalinclude:: /../saio/swift/object-server/4.conf ------------------------------------ Setting up scripts for running Swift ------------------------------------ #. Copy the SAIO scripts resetting the environment:: cd $HOME/swift/doc; cp -r saio/bin $HOME/bin; cd - chmod +x $HOME/bin/* #. Edit the ``$HOME/bin/resetswift`` script If you are using a loopback device substitute ``/dev/sdb1`` with ``/srv/swift-disk`` in the ``mkfs`` step:: sed -i "s/dev\/sdb1/srv\/swift-disk/" $HOME/bin/resetswift If you did not set up rsyslog for individual logging, remove the ``find /var/log/swift...`` line:: sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift On Fedora, replace ``service restart`` with ``systemctl restart .service``:: sed -i "s/service \(.*\) restart/systemctl restart \1.service/" $HOME/bin/resetswift #. Install the sample configuration file for running tests:: cp $HOME/swift/test/sample.conf /etc/swift/test.conf #. Add an environment variable for running tests below:: echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc #. Be sure that your ``PATH`` includes the ``bin`` directory:: echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc #. Source the above environment variables into your current environment:: . $HOME/.bashrc #. Construct the initial rings using the provided script:: remakerings You can expect the ouptut from this command to produce the following:: Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6020R127.0.0.1:6020/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6030R127.0.0.1:6030/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6040R127.0.0.1:6040/sdb4_"" with 1.0 weight got id 3 Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6021R127.0.0.1:6021/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6031R127.0.0.1:6031/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6041R127.0.0.1:6041/sdb4_"" with 1.0 weight got id 3 Reassigned 1024 (100.00%) partitions. Balance is now 0.00. Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0 Device d1r1z2-127.0.0.1:6022R127.0.0.1:6022/sdb2_"" with 1.0 weight got id 1 Device d2r1z3-127.0.0.1:6032R127.0.0.1:6032/sdb3_"" with 1.0 weight got id 2 Device d3r1z4-127.0.0.1:6042R127.0.0.1:6042/sdb4_"" with 1.0 weight got id 3 Reassigned 1024 (100.00%) partitions. Balance is now 0.00. #. Verify the unit tests run:: $HOME/swift/.unittests Note that the unit tests do not require any swift daemons running. #. Start the "main" Swift daemon processes (proxy, account, container, and object):: startmain (The "``Unable to increase file descriptor limit. Running as non-root?``" warnings are expected and ok.) #. Get an ``X-Storage-Url`` and ``X-Auth-Token``:: curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0 #. Check that you can ``GET`` account:: curl -v -H 'X-Auth-Token: ' #. Check that ``swift`` command provided by the python-swiftclient package works:: swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat #. Verify the functional tests run:: $HOME/swift/.functests (Note: functional tests will first delete everything in the configured accounts.) #. Verify the probe tests run:: $HOME/swift/.probetests (Note: probe tests will reset your environment as they call ``resetswift`` for each test.) ---------------- Debugging Issues ---------------- If all doesn't go as planned, and tests fail, or you can't auth, or something doesn't work, here are some good starting places to look for issues: #. Everything is logged using system facilities -- usually in ``/var/log/syslog``, but possibly in ``/var/log/messages`` on e.g. Fedora -- so that is a good first place to look for errors (most likely python tracebacks). #. Make sure all of the server processes are running. For the base functionality, the Proxy, Account, Container, and Object servers should be running. #. If one of the servers are not running, and no errors are logged to syslog, it may be useful to try to start the server manually, for example: ``swift-object-server /etc/swift/object-server/1.conf`` will start the object server. If there are problems not showing up in syslog, then you will likely see the traceback on startup. #. If you need to, you can turn off syslog for unit tests. This can be useful for environments where ``/dev/log`` is unavailable, or which cannot rate limit (unit tests generate a lot of logs very quickly). Open the file ``SWIFT_TEST_CONFIG_FILE`` points to, and change the value of ``fake_syslog`` to ``True``. swift-1.13.1/doc/source/middleware.rst0000664000175400017540000000746712323703611021017 0ustar jenkinsjenkins00000000000000.. _common_middleware: ********** Middleware ********** Account Quotas ============== .. automodule:: swift.common.middleware.account_quotas :members: :show-inheritance: .. _bulk: Bulk Operations (Delete and Archive Auto Extraction) ==================================================== .. automodule:: swift.common.middleware.bulk :members: :show-inheritance: .. _catch_errors: CatchErrors ============= .. automodule:: swift.common.middleware.catch_errors :members: :show-inheritance: CNAME Lookup ============ .. automodule:: swift.common.middleware.cname_lookup :members: :show-inheritance: .. _container-quotas: Container Quotas ================ .. automodule:: swift.common.middleware.container_quotas :members: :show-inheritance: .. _container-sync: Container Sync Middleware ========================= .. automodule:: swift.common.middleware.container_sync :members: :show-inheritance: Cross Domain Policies ===================== .. automodule:: swift.common.middleware.crossdomain :members: :show-inheritance: .. _discoverability: Discoverability =============== Swift will by default provide clients with an interface providing details about the installation. Unless disabled (i.e ``expose_info=false`` in :ref:`proxy-server-config`), a GET request to ``/info`` will return configuration data in JSON format. An example response:: {"swift": {"version": "1.11.0"}, "staticweb": {}, "tempurl": {}} This would signify to the client that swift version 1.11.0 is running and that staticweb and tempurl are available in this installation. There may be administrator-only information available via ``/info``. To retrieve it, one must use an HMAC-signed request, similar to TempURL. The signature may be produced like so:: swift-temp-url GET 3600 /info secret 2>/dev/null | sed s/temp_url/swiftinfo/g Domain Remap ============ .. automodule:: swift.common.middleware.domain_remap :members: :show-inheritance: Dynamic Large Objects ===================== .. automodule:: swift.common.middleware.dlo :members: :show-inheritance: .. _formpost: FormPost ======== .. automodule:: swift.common.middleware.formpost :members: :show-inheritance: .. _gatekeeper: GateKeeper ============= .. automodule:: swift.common.middleware.gatekeeper :members: :show-inheritance: .. _healthcheck: Healthcheck =========== .. automodule:: swift.common.middleware.healthcheck :members: :show-inheritance: KeystoneAuth ============ .. automodule:: swift.common.middleware.keystoneauth :members: :show-inheritance: List Endpoints ============== .. automodule:: swift.common.middleware.list_endpoints :members: :show-inheritance: Memcache ======== .. automodule:: swift.common.middleware.memcache :members: :show-inheritance: Name Check (Forbidden Character Filter) ======================================= .. automodule:: swift.common.middleware.name_check :members: :show-inheritance: Proxy Logging ============= .. automodule:: swift.common.middleware.proxy_logging :members: :show-inheritance: Ratelimit ========= .. automodule:: swift.common.middleware.ratelimit :members: :show-inheritance: .. _recon: Recon =========== .. automodule:: swift.common.middleware.recon :members: :show-inheritance: .. _slo-doc: Static Large Objects ==================== .. automodule:: swift.common.middleware.slo :members: :show-inheritance: .. _staticweb: StaticWeb ========= .. automodule:: swift.common.middleware.staticweb :members: :show-inheritance: .. _common_tempauth: TempAuth ======== .. automodule:: swift.common.middleware.tempauth :members: :show-inheritance: .. _tempurl: TempURL ======= .. automodule:: swift.common.middleware.tempurl :members: :show-inheritance: swift-1.13.1/doc/source/replication_network.rst0000664000175400017540000003265012323703611022754 0ustar jenkinsjenkins00000000000000.. _Dedicated-replication-network: ============================= Dedicated replication network ============================= ------- Summary ------- Swift's replication process is essential for consistency and availability of data. By default, replication activity will use the same network interface as other cluster operations. However, if a replication interface is set in the ring for a node, that node will send replication traffic on its designated separate replication network interface. Replication traffic includes REPLICATE requests and rsync traffic. To separate the cluster-internal replication traffic from client traffic, separate replication servers can be used. These replication servers are based on the standard storage servers, but they listen on the replication IP and only respond to REPLICATE requests. Storage servers can serve REPLICATE requests, so an operator can transition to using a separate replication network with no cluster downtime. Replication IP and port information is stored in the ring on a per-node basis. These parameters will be used if they are present, but they are not required. If this information does not exist or is empty for a particular node, the node's standard IP and port will be used for replication. -------------------- For SAIO replication -------------------- #. Create new script in ~/bin/ (for example: remakerings_new):: #!/bin/bash cd /etc/swift rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz swift-ring-builder object.builder create 18 3 1 swift-ring-builder object.builder add z1-127.0.0.1:6010R127.0.0.1:6050/sdb1 1 swift-ring-builder object.builder add z2-127.0.0.1:6020R127.0.0.1:6060/sdb2 1 swift-ring-builder object.builder add z3-127.0.0.1:6030R127.0.0.1:6070/sdb3 1 swift-ring-builder object.builder add z4-127.0.0.1:6040R127.0.0.1:6080/sdb4 1 swift-ring-builder object.builder rebalance swift-ring-builder container.builder create 18 3 1 swift-ring-builder container.builder add z1-127.0.0.1:6011R127.0.0.1:6051/sdb1 1 swift-ring-builder container.builder add z2-127.0.0.1:6021R127.0.0.1:6061/sdb2 1 swift-ring-builder container.builder add z3-127.0.0.1:6031R127.0.0.1:6071/sdb3 1 swift-ring-builder container.builder add z4-127.0.0.1:6041R127.0.0.1:6081/sdb4 1 swift-ring-builder container.builder rebalance swift-ring-builder account.builder create 18 3 1 swift-ring-builder account.builder add z1-127.0.0.1:6012R127.0.0.1:6052/sdb1 1 swift-ring-builder account.builder add z2-127.0.0.1:6022R127.0.0.1:6062/sdb2 1 swift-ring-builder account.builder add z3-127.0.0.1:6032R127.0.0.1:6072/sdb3 1 swift-ring-builder account.builder add z4-127.0.0.1:6042R127.0.0.1:6082/sdb4 1 swift-ring-builder account.builder rebalance .. note:: Syntax of adding device has been changed: R: was added between z-: and /_ . Added devices will use and for replication activities. #. Add next rows in /etc/rsyncd.conf:: [account6052] max connections = 25 path = /srv/1/node/ read only = false lock file = /var/lock/account6052.lock [account6062] max connections = 25 path = /srv/2/node/ read only = false lock file = /var/lock/account6062.lock [account6072] max connections = 25 path = /srv/3/node/ read only = false lock file = /var/lock/account6072.lock [account6082] max connections = 25 path = /srv/4/node/ read only = false lock file = /var/lock/account6082.lock [container6051] max connections = 25 path = /srv/1/node/ read only = false lock file = /var/lock/container6051.lock [container6061] max connections = 25 path = /srv/2/node/ read only = false lock file = /var/lock/container6061.lock [container6071] max connections = 25 path = /srv/3/node/ read only = false lock file = /var/lock/container6071.lock [container6081] max connections = 25 path = /srv/4/node/ read only = false lock file = /var/lock/container6081.lock [object6050] max connections = 25 path = /srv/1/node/ read only = false lock file = /var/lock/object6050.lock [object6060] max connections = 25 path = /srv/2/node/ read only = false lock file = /var/lock/object6060.lock [object6070] max connections = 25 path = /srv/3/node/ read only = false lock file = /var/lock/object6070.lock [object6080] max connections = 25 path = /srv/4/node/ read only = false lock file = /var/lock/object6080.lock #. Restart rsync deamon:: service rsync restart #. Add changes in configuration files in directories: * /etc/swift/object-server(files: 1.conf, 2.conf, 3.conf, 4.conf) * /etc/swift/container-server(files: 1.conf, 2.conf, 3.conf, 4.conf) * /etc/swift/account-server(files: 1.conf, 2.conf, 3.conf, 4.conf) delete all configuration options in section [<*>-replicator] #. Add configuration files for object-server, in /etc/swift/objec-server/ * 5.conf:: [DEFAULT] devices = /srv/1/node mount_check = false disable_fallocate = true bind_port = 6050 user = swift log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift [pipeline:main] pipeline = recon object-server [app:object-server] use = egg:swift#object replication_server = True [filter:recon] use = egg:swift#recon [object-replicator] vm_test_mode = yes * 6.conf:: [DEFAULT] devices = /srv/2/node mount_check = false disable_fallocate = true bind_port = 6060 user = swift log_facility = LOG_LOCAL3 recon_cache_path = /var/cache/swift2 [pipeline:main] pipeline = recon object-server [app:object-server] use = egg:swift#object replication_server = True [filter:recon] use = egg:swift#recon [object-replicator] vm_test_mode = yes * 7.conf:: [DEFAULT] devices = /srv/3/node mount_check = false disable_fallocate = true bind_port = 6070 user = swift log_facility = LOG_LOCAL4 recon_cache_path = /var/cache/swift3 [pipeline:main] pipeline = recon object-server [app:object-server] use = egg:swift#object replication_server = True [filter:recon] use = egg:swift#recon [object-replicator] vm_test_mode = yes * 8.conf:: [DEFAULT] devices = /srv/4/node mount_check = false disable_fallocate = true bind_port = 6080 user = swift log_facility = LOG_LOCAL5 recon_cache_path = /var/cache/swift4 [pipeline:main] pipeline = recon object-server [app:object-server] use = egg:swift#object replication_server = True [filter:recon] use = egg:swift#recon [object-replicator] vm_test_mode = yes #. Add configuration files for container-server, in /etc/swift/container-server/ * 5.conf:: [DEFAULT] devices = /srv/1/node mount_check = false disable_fallocate = true bind_port = 6051 user = swift log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift [pipeline:main] pipeline = recon container-server [app:container-server] use = egg:swift#container replication_server = True [filter:recon] use = egg:swift#recon [container-replicator] vm_test_mode = yes * 6.conf:: [DEFAULT] devices = /srv/2/node mount_check = false disable_fallocate = true bind_port = 6061 user = swift log_facility = LOG_LOCAL3 recon_cache_path = /var/cache/swift2 [pipeline:main] pipeline = recon container-server [app:container-server] use = egg:swift#container replication_server = True [filter:recon] use = egg:swift#recon [container-replicator] vm_test_mode = yes * 7.conf:: [DEFAULT] devices = /srv/3/node mount_check = false disable_fallocate = true bind_port = 6071 user = swift log_facility = LOG_LOCAL4 recon_cache_path = /var/cache/swift3 [pipeline:main] pipeline = recon container-server [app:container-server] use = egg:swift#container replication_server = True [filter:recon] use = egg:swift#recon [container-replicator] vm_test_mode = yes * 8.conf:: [DEFAULT] devices = /srv/4/node mount_check = false disable_fallocate = true bind_port = 6081 user = swift log_facility = LOG_LOCAL5 recon_cache_path = /var/cache/swift4 [pipeline:main] pipeline = recon container-server [app:container-server] use = egg:swift#container replication_server = True [filter:recon] use = egg:swift#recon [container-replicator] vm_test_mode = yes #. Add configuration files for account-server, in /etc/swift/account-server/ * 5.conf:: [DEFAULT] devices = /srv/1/node mount_check = false disable_fallocate = true bind_port = 6052 user = swift log_facility = LOG_LOCAL2 recon_cache_path = /var/cache/swift [pipeline:main] pipeline = recon account-server [app:account-server] use = egg:swift#account replication_server = True [filter:recon] use = egg:swift#recon [account-replicator] vm_test_mode = yes * 6.conf:: [DEFAULT] devices = /srv/2/node mount_check = false disable_fallocate = true bind_port = 6062 user = swift log_facility = LOG_LOCAL3 recon_cache_path = /var/cache/swift2 [pipeline:main] pipeline = recon account-server [app:account-server] use = egg:swift#account replication_server = True [filter:recon] use = egg:swift#recon [account-replicator] vm_test_mode = yes * 7.conf:: [DEFAULT] devices = /srv/3/node mount_check = false disable_fallocate = true bind_port = 6072 user = swift log_facility = LOG_LOCAL4 recon_cache_path = /var/cache/swift3 [pipeline:main] pipeline = recon account-server [app:account-server] use = egg:swift#account replication_server = True [filter:recon] use = egg:swift#recon [account-replicator] vm_test_mode = yes * 8.conf:: [DEFAULT] devices = /srv/4/node mount_check = false disable_fallocate = true bind_port = 6082 user = swift log_facility = LOG_LOCAL5 recon_cache_path = /var/cache/swift4 [pipeline:main] pipeline = recon account-server [app:account-server] use = egg:swift#account replication_server = True [filter:recon] use = egg:swift#recon [account-replicator] vm_test_mode = yes --------------------------------- For a Multiple Server replication --------------------------------- #. Move configuration file. * Configuration file for object-server from /etc/swift/object-server.conf to /etc/swift/object-server/1.conf * Configuration file for container-server from /etc/swift/container-server.conf to /etc/swift/container-server/1.conf * Configuration file for account-server from /etc/swift/account-server.conf to /etc/swift/account-server/1.conf #. Add changes in configuration files in directories: * /etc/swift/object-server(files: 1.conf) * /etc/swift/container-server(files: 1.conf) * /etc/swift/account-server(files: 1.conf) delete all configuration options in section [<*>-replicator] #. Add configuration files for object-server, in /etc/swift/object-server/2.conf:: [DEFAULT] bind_ip = $STORAGE_LOCAL_NET_IP workers = 2 [pipeline:main] pipeline = object-server [app:object-server] use = egg:swift#object replication_server = True [object-replicator] #. Add configuration files for container-server, in /etc/swift/container-server/2.conf:: [DEFAULT] bind_ip = $STORAGE_LOCAL_NET_IP workers = 2 [pipeline:main] pipeline = container-server [app:container-server] use = egg:swift#container replication_server = True [container-replicator] #. Add configuration files for account-server, in /etc/swift/account-server/2.conf:: [DEFAULT] bind_ip = $STORAGE_LOCAL_NET_IP workers = 2 [pipeline:main] pipeline = account-server [app:account-server] use = egg:swift#account replication_server = True [account-replicator] swift-1.13.1/doc/source/development_ondisk_backends.rst0000664000175400017540000000267312323703611024417 0ustar jenkinsjenkins00000000000000=============================== Pluggable On-Disk Back-end APIs =============================== The internal REST API used between the proxy server and the account, container and object server is almost identical to public Swift REST API, but with a few internal extentsions (for example, update an account with a new container). The pluggable back-end APIs for the three REST API servers (account, container, object) abstracts the needs for servicing the various REST APIs from the details of how data is laid out and stored on-disk. The APIs are documented in the reference implementations for all three servers. For historical reasons, the object server backend reference implementation module is named `diskfile`, while the account and container server backend reference implementation modules are named appropriately. This API is still under development and not yet finalized. ----------------------------------------- Back-end API for Account Server REST APIs ----------------------------------------- .. automodule:: swift.account.backend :noindex: :members: ------------------------------------------- Back-end API for Container Server REST APIs ------------------------------------------- .. automodule:: swift.container.backend :noindex: :members: ---------------------------------------- Back-end API for Object Server REST APIs ---------------------------------------- .. automodule:: swift.obj.diskfile :noindex: :members: swift-1.13.1/doc/source/container.rst0000664000175400017540000000144412323703611020651 0ustar jenkinsjenkins00000000000000.. _Container: ********* Container ********* .. _container-auditor: Container Auditor ================= .. automodule:: swift.container.auditor :members: :undoc-members: :show-inheritance: .. _container-backend: Container Backend ================= .. automodule:: swift.container.backend :members: :undoc-members: :show-inheritance: .. _container-server: Container Server ================ .. automodule:: swift.container.server :members: :undoc-members: :show-inheritance: Container Sync ============== .. automodule:: swift.container.sync :members: :undoc-members: :show-inheritance: .. _container-updater: Container Updater ================= .. automodule:: swift.container.updater :members: :undoc-members: :show-inheritance: swift-1.13.1/doc/source/test-cors.html0000664000175400017540000000372512323703611020752 0ustar jenkinsjenkins00000000000000 Test CORS Token


Method


URL (Container or Object)



    

    

    


    

  

swift-1.13.1/doc/source/getting_started.rst0000664000175400017540000000271412323703611022057 0ustar  jenkinsjenkins00000000000000===============
Getting Started
===============

-------------------
System Requirements
-------------------

Swift development currently targets Ubuntu Server 10.04, but should work on 
most Linux platforms with the following software:

* Python 2.6
* rsync 3.0

And the following python libraries:

* Eventlet 0.9.8
* Setuptools
* Simplejson
* Xattr
* Nose
* Sphinx
* Netifaces
* Dnspython
* Pastedeploy


-------------
Getting Swift
-------------

Swift's source code is hosted on github and managed with git.  The current trunk can be checked out like this:

    ``git clone https://github.com/openstack/swift.git``

A source tarball for the latest release of Swift is available on the `launchpad project page `_.

Prebuilt packages for Ubuntu are available starting with Natty, or from PPAs for earlier releases.

* `Swift Ubuntu Packages `_
* `Swift PPA Archive `_

-----------
Development
-----------

To get started with development with Swift, or to just play around, the
following docs will be useful:

* :doc:`Swift All in One ` - Set up a VM with Swift installed
* :doc:`Development Guidelines `

----------
Production
----------

If you want to set up and configure Swift for a production cluster, the following doc should be useful:

* :doc:`Multiple Server Swift Installation `
swift-1.13.1/doc/source/object.rst0000664000175400017540000000170012323703611020130 0ustar  jenkinsjenkins00000000000000.. _object:

******
Object
******

.. _object-auditor:

Object Auditor
==============

.. automodule:: swift.obj.auditor
    :members:
    :undoc-members:
    :show-inheritance:

.. _object-diskfile:

Object Backend
==============

.. automodule:: swift.obj.diskfile
    :members:
    :undoc-members:
    :show-inheritance:

.. _object-replicator:

Object Replicator
=================

.. automodule:: swift.obj.replicator
    :members:
    :undoc-members:
    :show-inheritance:

.. automodule:: swift.obj.ssync_sender
    :members:
    :undoc-members:
    :show-inheritance:

.. automodule:: swift.obj.ssync_receiver
    :members:
    :undoc-members:
    :show-inheritance:

.. _object-server:

Object Server
=============

.. automodule:: swift.obj.server
    :members:
    :undoc-members:
    :show-inheritance:

.. _object-updater:

Object Updater
==============

.. automodule:: swift.obj.updater
    :members:
    :undoc-members:
    :show-inheritance:
swift-1.13.1/doc/source/ratelimit.rst0000664000175400017540000001214712323703611020663 0ustar  jenkinsjenkins00000000000000=============
Rate Limiting
=============

Rate limiting in swift is implemented as a pluggable middleware.  Rate
limiting is performed on requests that result in database writes to the
account and container sqlite dbs.  It uses memcached and is dependent on
the proxy servers having highly synchronized time.  The rate limits are
limited by the accuracy of the proxy server clocks.

--------------
Configuration
--------------

All configuration is optional.  If no account or container limits are provided
there will be no rate limiting.  Configuration available:

================================ ======= ======================================
Option                           Default Description
-------------------------------- ------- --------------------------------------
clock_accuracy                   1000    Represents how accurate the proxy
                                         servers' system clocks are with each
                                         other. 1000 means that all the
                                         proxies' clock are accurate to each
                                         other within 1 millisecond. No
                                         ratelimit should be higher than the
                                         clock accuracy.
max_sleep_time_seconds           60      App will immediately return a 498
                                         response if the necessary sleep time
                                         ever exceeds the given
                                         max_sleep_time_seconds.
log_sleep_time_seconds           0       To allow visibility into rate limiting
                                         set this value > 0 and all sleeps
                                         greater than the number will be
                                         logged.
rate_buffer_seconds              5       Number of seconds the rate counter can
                                         drop and be allowed to catch up (at a
                                         faster than listed rate). A larger
                                         number will result in larger spikes in
                                         rate but better average accuracy.
account_ratelimit                0       If set, will limit PUT and DELETE
                                         requests to
                                         /account_name/container_name. Number
                                         is in requests per second.
account_whitelist                ''      Comma separated lists of account names
                                         that will not be rate limited.
account_blacklist                ''      Comma separated lists of account names
                                         that will not be allowed. Returns a
                                         497 response.
container_ratelimit_size         ''      When set with container_ratelimit_x =
                                         r: for containers of size x, limit
                                         requests per second to r. Will limit
                                         PUT, DELETE, and POST requests to
                                         /a/c/o.
container_listing_ratelimit_size ''      When set with
                                         container_listing_ratelimit_x = r: for
                                         containers of size x, limit listing
                                         requests per second to r. Will limit
                                         GET requests to /a/c.
================================ ======= ======================================

The container rate limits are linearly interpolated from the values given.  A
sample container rate limiting could be:

container_ratelimit_100 = 100

container_ratelimit_200 = 50

container_ratelimit_500 = 20

This would result in

================    ============
Container Size      Rate Limit
----------------    ------------
0-99                No limiting
100                 100
150                 75
500                 20
1000                20
================    ============


-----------------------------
Account Specific Ratelimiting
-----------------------------

The above ratelimiting is to prevent the "many writes to a single container"
bottleneck from causing a problem. There could also be a problem where a single
account is just using too much of the cluster's resources.  In this case, the
container ratelimits may not help because the customer could be doing thousands
of reqs/sec to distributed containers each getting a small fraction of the
total so those limits would never trigger. If a system administrator notices
this, he/she can set the X-Account-Sysmeta-Global-Write-Ratelimit on an account
and that will limit the total number of write requests (PUT, POST, DELETE,
COPY) that account can do for the whole account. This limit will be in addition
to the applicable account/container limits from above. This header will be
hidden from the user, because of the gatekeeper middleware, and can only be set
using a direct client to the account nodes. It accepts a float value and will
only limit requests if the value is > 0.
swift-1.13.1/doc/source/overview_large_objects.rst0000664000175400017540000002405512323703611023423 0ustar  jenkinsjenkins00000000000000.. _large-objects:

====================
Large Object Support
====================

--------
Overview
--------

Swift has a limit on the size of a single uploaded object; by default this is
5GB. However, the download size of a single object is virtually unlimited with
the concept of segmentation. Segments of the larger object are uploaded and a
special manifest file is created that, when downloaded, sends all the segments
concatenated as a single object. This also offers much greater upload speed
with the possibility of parallel uploads of the segments.

.. _dynamic-large-objects:

---------------------
Dynamic Large Objects
---------------------

---------------
Using ``swift``
---------------

The quickest way to try out this feature is use the ``swift`` Swift Tool
included with the `python-swiftclient`_ library.  You can use the ``-S``
option to specify the segment size to use when splitting a large file. For
example::

    swift upload test_container -S 1073741824 large_file

This would split the large_file into 1G segments and begin uploading those
segments in parallel. Once all the segments have been uploaded, ``swift`` will
then create the manifest file so the segments can be downloaded as one.

So now, the following ``swift`` command would download the entire large object::

    swift download test_container large_file

``swift`` uses a strict convention for its segmented object
support. In the above example it will upload all the segments into a
second container named test_container_segments. These segments will
have names like large_file/1290206778.25/21474836480/00000000,
large_file/1290206778.25/21474836480/00000001, etc.

The main benefit for using a separate container is that the main container
listings will not be polluted with all the segment names. The reason for using
the segment name format of /// is so that an
upload of a new file with the same name won't overwrite the contents of the
first until the last moment when the manifest file is updated.

``swift`` will manage these segment files for you, deleting old segments on
deletes and overwrites, etc. You can override this behavior with the
``--leave-segments`` option if desired; this is useful if you want to have
multiple versions of the same large object available.

.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient

----------
Direct API
----------

You can also work with the segments and manifests directly with HTTP
requests instead of having ``swift`` do that for you. You can just
upload the segments like you would any other object and the manifest
is just a zero-byte file with an extra ``X-Object-Manifest`` header.

All the object segments need to be in the same container, have a common object
name prefix, and their names sort in the order they should be concatenated.
They don't have to be in the same container as the manifest file will be, which
is useful to keep container listings clean as explained above with ``swift``.

The manifest file is simply a zero-byte file with the extra
``X-Object-Manifest: /`` header, where ```` is
the container the object segments are in and ```` is the common prefix
for all the segments.

It is best to upload all the segments first and then create or update the
manifest. In this way, the full object won't be available for downloading until
the upload is complete. Also, you can upload a new set of segments to a second
location and then update the manifest to point to this new location. During the
upload of the new segments, the original manifest will still be available to
download the first set of segments.

Here's an example using ``curl`` with tiny 1-byte segments::

    # First, upload the segments
    curl -X PUT -H 'X-Auth-Token: ' \
        http:///container/myobject/1 --data-binary '1'
    curl -X PUT -H 'X-Auth-Token: ' \
        http:///container/myobject/2 --data-binary '2'
    curl -X PUT -H 'X-Auth-Token: ' \
        http:///container/myobject/3 --data-binary '3'

    # Next, create the manifest file
    curl -X PUT -H 'X-Auth-Token: ' \
        -H 'X-Object-Manifest: container/myobject/' \
        http:///container/myobject --data-binary ''

    # And now we can download the segments as a single object
    curl -H 'X-Auth-Token: ' \
        http:///container/myobject

.. _static-large-objects:

--------------------
Static Large Objects
--------------------

----------
Direct API
----------

SLO support centers around the user generated manifest file. After the user
has uploaded the segments into their account a manifest file needs to be
built and uploaded. All object segments, except the last, must be above 1 MB
(by default) in size. Please see the SLO docs for :ref:`slo-doc` further
details.

----------------
Additional Notes
----------------

* With a ``GET`` or ``HEAD`` of a manifest file, the ``X-Object-Manifest:
  /`` header will be returned with the concatenated object
  so you can tell where it's getting its segments from.

* The response's ``Content-Length`` for a ``GET`` or ``HEAD`` on the manifest
  file will be the sum of all the segments in the ``/``
  listing, dynamically. So, uploading additional segments after the manifest is
  created will cause the concatenated object to be that much larger; there's no
  need to recreate the manifest file.

* The response's ``Content-Type`` for a ``GET`` or ``HEAD`` on the manifest
  will be the same as the ``Content-Type`` set during the ``PUT`` request that
  created the manifest. You can easily change the ``Content-Type`` by reissuing
  the ``PUT``.

* The response's ``ETag`` for a ``GET`` or ``HEAD`` on the manifest file will
  be the MD5 sum of the concatenated string of ETags for each of the segments
  in the manifest (for DLO, from the listing ``/``).
  Usually in Swift the ETag is the MD5 sum of the contents of the object, and
  that holds true for each segment independently. But it's not meaningful to
  generate such an ETag for the manifest itself so this method was chosen to
  at least offer change detection.


.. note::

    If you are using the container sync feature you will need to ensure both
    your manifest file and your segment files are synced if they happen to be
    in different containers.

-------
History
-------

Dynamic large object support has gone through various iterations before
settling on this implementation.

The primary factor driving the limitation of object size in swift is
maintaining balance among the partitions of the ring.  To maintain an even
dispersion of disk usage throughout the cluster the obvious storage pattern
was to simply split larger objects into smaller segments, which could then be
glued together during a read.

Before the introduction of large object support some applications were already
splitting their uploads into segments and re-assembling them on the client
side after retrieving the individual pieces.  This design allowed the client
to support backup and archiving of large data sets, but was also frequently
employed to improve performance or reduce errors due to network interruption.
The major disadvantage of this method is that knowledge of the original
partitioning scheme is required to properly reassemble the object, which is
not practical for some use cases, such as CDN origination.

In order to eliminate any barrier to entry for clients wanting to store
objects larger than 5GB, initially we also prototyped fully transparent
support for large object uploads.  A fully transparent implementation would
support a larger max size by automatically splitting objects into segments
during upload within the proxy without any changes to the client API.  All
segments were completely hidden from the client API.

This solution introduced a number of challenging failure conditions into the
cluster, wouldn't provide the client with any option to do parallel uploads,
and had no basis for a resume feature.  The transparent implementation was
deemed just too complex for the benefit.

The current "user manifest" design was chosen in order to provide a
transparent download of large objects to the client and still provide the
uploading client a clean API to support segmented uploads.

To meet an many use cases as possible swift supports two types of large
object manifests. Dynamic and static large object manifests both support
the same idea of allowing the user to upload many segments to be later
downloaded as a single file.

Dynamic large objects rely on a container lising to provide the manifest.
This has the advantage of allowing the user to add/removes segments from the
manifest at any time. It has the disadvantage of relying on eventually
consistent container listings. All three copies of the container dbs must
be updated for a complete list to be guaranteed. Also, all segments must
be in a single container, which can limit concurrent upload speed.

Static large objects rely on a user provided manifest file. A user can
upload objects into multiple containers and then reference those objects
(segments) in a self generated manifest file. Future GETs to that file will
download the concatenation of the specified segments. This has the advantage of
being able to immediately download the complete object once the manifest has
been successfully PUT. Being able to upload segments into separate containers
also improves concurrent upload speed. It has the disadvantage that the
manifest is finalized once PUT. Any changes to it means it has to be replaced.

Between these two methods the user has great flexibility in how (s)he chooses
to upload and retrieve large objects to swift. Swift does not, however, stop
the user from harming themselves. In both cases the segments are deletable by
the user at any time. If a segment was deleted by mistake, a dynamic large
object, having no way of knowing it was ever there, would happily ignore the
deleted file and the user will get an incomplete file. A static large object
would, when failing to retrieve the object specified in the manifest, drop the
connection and the user would receive partial results.
swift-1.13.1/doc/source/deployment_guide.rst0000664000175400017540000017310012323703611022223 0ustar  jenkinsjenkins00000000000000================
Deployment Guide
================

-----------------------
Hardware Considerations
-----------------------

Swift is designed to run on commodity hardware. At Rackspace, our storage
servers are currently running fairly generic 4U servers with 24 2T SATA
drives and 8 cores of processing power. RAID on the storage drives is not
required and not recommended. Swift's disk usage pattern is the worst
case possible for RAID, and performance degrades very quickly using RAID 5
or 6.

------------------
Deployment Options
------------------

The swift services run completely autonomously, which provides for a lot of
flexibility when architecting the hardware deployment for swift. The 4 main
services are:

#. Proxy Services
#. Object Services
#. Container Services
#. Account Services

The Proxy Services are more CPU and network I/O intensive. If you are using
10g networking to the proxy, or are terminating SSL traffic at the proxy,
greater CPU power will be required.

The Object, Container, and Account Services (Storage Services) are more disk
and network I/O intensive.

The easiest deployment is to install all services on each server. There is
nothing wrong with doing this, as it scales each service out horizontally.

At Rackspace, we put the Proxy Services on their own servers and all of the
Storage Services on the same server. This allows us to send 10g networking to
the proxy and 1g to the storage servers, and keep load balancing to the
proxies more manageable.  Storage Services scale out horizontally as storage
servers are added, and we can scale overall API throughput by adding more
Proxies.

If you need more throughput to either Account or Container Services, they may
each be deployed to their own servers. For example you might use faster (but
more expensive) SAS or even SSD drives to get faster disk I/O to the databases.

Load balancing and network design is left as an exercise to the reader,
but this is a very important part of the cluster, so time should be spent
designing the network for a Swift cluster.


---------------------
Web Front End Options
---------------------

Swift comes with an integral web front end. However, it can also be deployed
as a request processor of an Apache2 using mod_wsgi as described in
:doc:`Apache Deployment Guide `.

.. _ring-preparing:

------------------
Preparing the Ring
------------------

The first step is to determine the number of partitions that will be in the
ring. We recommend that there be a minimum of 100 partitions per drive to
insure even distribution across the drives. A good starting point might be
to figure out the maximum number of drives the cluster will contain, and then
multiply by 100, and then round up to the nearest power of two.

For example, imagine we are building a cluster that will have no more than
5,000 drives. That would mean that we would have a total number of 500,000
partitions, which is pretty close to 2^19, rounded up.

It is also a good idea to keep the number of partitions small (relatively).
The more partitions there are, the more work that has to be done by the
replicators and other backend jobs and the more memory the rings consume in
process. The goal is to find a good balance between small rings and maximum
cluster size.

The next step is to determine the number of replicas to store of the data.
Currently it is recommended to use 3 (as this is the only value that has
been tested). The higher the number, the more storage that is used but the
less likely you are to lose data.

It is also important to determine how many zones the cluster should have. It is
recommended to start with a minimum of 5 zones. You can start with fewer, but
our testing has shown that having at least five zones is optimal when failures
occur. We also recommend trying to configure the zones at as high a level as
possible to create as much isolation as possible. Some example things to take
into consideration can include physical location, power availability, and
network connectivity. For example, in a small cluster you might decide to
split the zones up by cabinet, with each cabinet having its own power and
network connectivity. The zone concept is very abstract, so feel free to use
it in whatever way best isolates your data from failure. Zones are referenced
by number, beginning with 1.

You can now start building the ring with::

    swift-ring-builder  create   

This will start the ring build process creating the  with
2^ partitions.  is the time in hours before a
specific partition can be moved in succession (24 is a good value for this).

Devices can be added to the ring with::

    swift-ring-builder  add z-:/_ 

This will add a device to the ring where  is the name of the
builder file that was created previously,  is the number of the zone
this device is in,  is the ip address of the server the device is in,
 is the port number that the server is running on,  is
the name of the device on the server (for example: sdb1),  is a string
of metadata for the device (optional), and  is a float weight that
determines how many partitions are put on the device relative to the rest of
the devices in the cluster (a good starting point is 100.0 x TB on the drive).
Add each device that will be initially in the cluster.

Once all of the devices are added to the ring, run::

    swift-ring-builder  rebalance

This will distribute the partitions across the drives in the ring. It is
important whenever making changes to the ring to make all the changes
required before running rebalance. This will ensure that the ring stays as
balanced as possible, and as few partitions are moved as possible.

The above process should be done to make a ring for each storage service
(Account, Container and Object). The builder files will be needed in future
changes to the ring, so it is very important that these be kept and backed up.
The resulting .tar.gz ring file should be pushed to all of the servers in the
cluster. For more information about building rings, running
swift-ring-builder with no options will display help text with available
commands and options. More information on how the ring works internally
can be found in the :doc:`Ring Overview `.

.. _general-service-configuration:

-----------------------------
General Service Configuration
-----------------------------

Most Swift services fall into two categories.  Swift's wsgi servers and
background daemons.  

For more information specific to the configuration of Swift's wsgi servers
with paste deploy see :ref:`general-server-configuration`

Configuration for servers and daemons can be expressed together in the same
file for each type of server, or separately.  If a required section for the
service trying to start is missing there will be an error.  The sections not
used by the service are ignored.

Consider the example of an object storage node.  By convention configuration
for the object-server, object-updater, object-replicator, and object-auditor
exist in a single file ``/etc/swift/object-server.conf``::

    [DEFAULT]

    [pipeline:main]
    pipeline = object-server

    [app:object-server]
    use = egg:swift#object

    [object-replicator]
    reclaim_age = 259200

    [object-updater]

    [object-auditor]

Swift services expect a configuration path as the first argument::

    $ swift-object-auditor 
    Usage: swift-object-auditor CONFIG [options]

    Error: missing config path argument

If you omit the object-auditor section this file could not be used as the
configuration path when starting the ``swift-object-auditor`` daemon::

    $ swift-object-auditor /etc/swift/object-server.conf 
    Unable to find object-auditor config section in /etc/swift/object-server.conf

If the configuration path is a directory instead of a file all of the files in
the directory with the file extension ".conf" will be combined to generate the
configuration object which is delivered to the Swift service.  This is
referred to generally as "directory based configuration".

Directory based configuration leverages ConfigParser's native multi-file
support.  Files ending in ".conf" in the given directory are parsed in
lexicographical order.  Filenames starting with '.' are ignored.  A mixture of
file and directory configuration paths is not supported - if the configuration
path is a file only that file will be parsed.

The swift service management tool ``swift-init`` has adopted the convention of
looking for ``/etc/swift/{type}-server.conf.d/`` if the file
``/etc/swift/{type}-server.conf`` file does not exist.

When using directory based configuration, if the same option under the same
section appears more than once in different files, the last value parsed is
said to override previous occurrences.  You can ensure proper override
precedence by prefixing the files in the configuration directory with
numerical values.::

    /etc/swift/
        default.base
        object-server.conf.d/
            000_default.conf -> ../default.base
            001_default-override.conf
            010_server.conf
            020_replicator.conf
            030_updater.conf
            040_auditor.conf

You can inspect the resulting combined configuration object using the
``swift-config`` command line tool

.. _general-server-configuration:

----------------------------
General Server Configuration
----------------------------

Swift uses paste.deploy (http://pythonpaste.org/deploy/) to manage server
configurations.

Default configuration options are set in the `[DEFAULT]` section, and any
options specified there can be overridden in any of the other sections BUT
ONLY BY USING THE SYNTAX ``set option_name = value``. This is the unfortunate
way paste.deploy works and I'll try to explain it in full.

First, here's an example paste.deploy configuration file::

    [DEFAULT]
    name1 = globalvalue
    name2 = globalvalue
    name3 = globalvalue
    set name4 = globalvalue

    [pipeline:main]
    pipeline = myapp

    [app:myapp]
    use = egg:mypkg#myapp
    name2 = localvalue
    set name3 = localvalue
    set name5 = localvalue
    name6 = localvalue

The resulting configuration that myapp receives is::

    global {'__file__': '/etc/mypkg/wsgi.conf', 'here': '/etc/mypkg',
            'name1': 'globalvalue',
            'name2': 'globalvalue',
            'name3': 'localvalue',
            'name4': 'globalvalue',
            'name5': 'localvalue',
            'set name4': 'globalvalue'}
    local {'name6': 'localvalue'}

So, `name1` got the global value which is fine since it's only in the `DEFAULT`
section anyway.

`name2` got the global value from `DEFAULT` even though it appears to be
overridden in the `app:myapp` subsection. This is just the unfortunate way
paste.deploy works (at least at the time of this writing.)

`name3` got the local value from the `app:myapp` subsection because it is using
the special paste.deploy syntax of ``set option_name = value``. So, if you want
a default value for most app/filters but want to overridde it in one
subsection, this is how you do it.

`name4` got the global value from `DEFAULT` since it's only in that section
anyway. But, since we used the ``set`` syntax in the `DEFAULT` section even
though we shouldn't, notice we also got a ``set name4`` variable. Weird, but
probably not harmful.

`name5` got the local value from the `app:myapp` subsection since it's only
there anyway, but notice that it is in the global configuration and not the
local configuration. This is because we used the ``set`` syntax to set the
value. Again, weird, but not harmful since Swift just treats the two sets of
configuration values as one set anyway.

`name6` got the local value from `app:myapp` subsection since it's only there,
and since we didn't use the ``set`` syntax, it's only in the local
configuration and not the global one. Though, as indicated above, there is no
special distinction with Swift.

That's quite an explanation for something that should be so much simpler, but
it might be important to know how paste.deploy interprets configuration files.
The main rule to remember when working with Swift configuration files is:

.. note::

    Use the ``set option_name = value`` syntax in subsections if the option is
    also set in the ``[DEFAULT]`` section. Don't get in the habit of always
    using the ``set`` syntax or you'll probably mess up your non-paste.deploy
    configuration files.

--------------------
Common configuration
--------------------

An example of common configuration file can be found at etc/swift.conf-sample

The following configuration options are available:

===================  ==========  =============================================
Option               Default     Description
-------------------  ----------  ---------------------------------------------
max_header_size      8192        max_header_size is the max number of bytes in
                                 the utf8 encoding of each header. Using 8192
                                 as default because eventlet use 8192 as max
                                 size of header line. This value may need to
                                 be increased when using identity v3 API
                                 tokens including more than 7 catalog entries.
                                 See also include_service_catalog in
                                 proxy-server.conf-sample (documented in
                                 overview_auth.rst)
===================  ==========  =============================================

---------------------------
Object Server Configuration
---------------------------

An Example Object Server configuration can be found at
etc/object-server.conf-sample in the source code repository.

The following configuration options are available:

[DEFAULT]

===================  ==========  =============================================
Option               Default     Description
-------------------  ----------  ---------------------------------------------
swift_dir            /etc/swift  Swift configuration directory
devices              /srv/node   Parent directory of where devices are mounted
mount_check          true        Whether or not check if the devices are
                                 mounted to prevent accidentally writing
                                 to the root device
bind_ip              0.0.0.0     IP Address for server to bind to
bind_port            6000        Port for server to bind to
bind_timeout         30          Seconds to attempt bind before giving up
workers              auto        Override the number of pre-forked workers
                                 that will accept connections.  If set it
                                 should be an integer, zero means no fork.  If
                                 unset, it will try to default to the number
                                 of effective cpu cores and fallback to one.
                                 Increasing the number of workers may reduce
                                 the possibility of slow file system
                                 operations in one request from negatively
                                 impacting other requests, but may not be as
                                 efficient as tuning :ref:`threads_per_disk
                                 `
max_clients          1024        Maximum number of clients one worker can
                                 process simultaneously (it will actually
                                 accept(2) N + 1). Setting this to one (1)
                                 will only handle one request at a time,
                                 without accepting another request
                                 concurrently.
disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                 underlying filesystem does not support it.
log_custom_handlers  None        Comma-separated list of functions to call
                                 to setup custom log handlers.
eventlet_debug       false       If true, turn on debug logging for eventlet
fallocate_reserve    0           You can set fallocate_reserve to the number of
                                 bytes you'd like fallocate to reserve, whether
                                 there is space for the given file size or not.
                                 This is useful for systems that behave badly
                                 when they completely run out of space; you can
                                 make the services pretend they're out of space
                                 early.
conn_timeout         0.5         Time to wait while attempting to connect to
                                 another backend node.
node_timeout         3           Time to wait while sending each chunk of data
                                 to another backend node.
client_timeout       60          Time to wait while receiving each chunk of
                                 data from a client or another backend node.
network_chunk_size   65536       Size of chunks to read/write over the network
disk_chunk_size      65536       Size of chunks to read/write to disk
===================  ==========  =============================================

.. _object-server-options:

[object-server]

=============================  =============  =================================
Option                         Default        Description
-----------------------------  -------------  ---------------------------------
use                                           paste.deploy entry point for the
                                              object server.  For most cases,
                                              this should be
                                              `egg:swift#object`.
set log_name                   object-server  Label used when logging
set log_facility               LOG_LOCAL0     Syslog log facility
set log_level                  INFO           Logging level
set log_requests               True           Whether or not to log each
                                              request
user                           swift          User to run as
max_upload_time                86400          Maximum time allowed to upload an
                                              object
slow                           0              If > 0, Minimum time in seconds
                                              for a PUT or DELETE request to
                                              complete
mb_per_sync                    512            On PUT requests, sync file every
                                              n MB
keep_cache_size                5242880        Largest object size to keep in
                                              buffer cache
keep_cache_private             false          Allow non-public objects to stay
                                              in kernel's buffer cache
threads_per_disk               0              Size of the per-disk thread pool
                                              used for performing disk I/O. The
                                              default of 0 means to not use a
                                              per-disk thread pool. It is
                                              recommended to keep this value
                                              small, as large values can result
                                              in high read latencies due to
                                              large queue depths. A good
                                              starting point is 4 threads per
                                              disk.
replication_concurrency        4              Set to restrict the number of
                                              concurrent incoming REPLICATION
                                              requests; set to 0 for unlimited
replication_one_per_device     True           Restricts incoming REPLICATION
                                              requests to one per device,
                                              replication_currency above
                                              allowing. This can help control
                                              I/O to each device, but you may
                                              wish to set this to False to
                                              allow multiple REPLICATION
                                              requests (up to the above
                                              replication_concurrency setting)
                                              per device.
replication_lock_timeout       15             Number of seconds to wait for an
                                              existing replication device lock
                                              before giving up.
replication_failure_threshold  100            The number of subrequest failures
                                              before the
                                              replication_failure_ratio is
                                              checked
replication_failure_ratio      1.0            If the value of failures /
                                              successes of REPLICATION
                                              subrequests exceeds this ratio,
                                              the overall REPLICATION request
                                              will be aborted
=============================  =============  =================================

[object-replicator]

==================  =================  =======================================
Option              Default            Description
------------------  -----------------  ---------------------------------------
log_name            object-replicator  Label used when logging
log_facility        LOG_LOCAL0         Syslog log facility
log_level           INFO               Logging level
daemonize           yes                Whether or not to run replication as a
                                       daemon
run_pause           30                 Time in seconds to wait between
                                       replication passes
concurrency         1                  Number of replication workers to spawn
timeout             5                  Timeout value sent to rsync --timeout
                                       and --contimeout options
stats_interval      3600               Interval in seconds between logging
                                       replication statistics
reclaim_age         604800             Time elapsed in seconds before an
                                       object can be reclaimed
handoffs_first      false              If set to True, partitions that are
                                       not supposed to be on the node will be
                                       replicated first.  The default setting
                                       should not be changed, except for
                                       extreme situations.
handoff_delete      auto               By default handoff partitions will be
                                       removed when it has successfully
                                       replicated to all the canonical nodes.
                                       If set to an integer n, it will remove
                                       the partition if it is successfully
                                       replicated to n nodes.  The default
                                       setting should not be changed, except
                                       for extremem situations.
node_timeout        DEFAULT or 10      Request timeout to external services.
                                       This uses what's set here, or what's set
                                       in the DEFAULT section, or 10 (though
                                       other sections use 3 as the final
                                       default).
==================  =================  =======================================

[object-updater]

==================  ==============  ==========================================
Option              Default         Description
------------------  --------------  ------------------------------------------
log_name            object-updater  Label used when logging
log_facility        LOG_LOCAL0      Syslog log facility
log_level           INFO            Logging level
interval            300             Minimum time for a pass to take
concurrency         1               Number of updater workers to spawn
node_timeout        DEFAULT or 10   Request timeout to external services. This
                                    uses what's set here, or what's set in the
                                    DEFAULT section, or 10 (though other
                                    sections use 3 as the final default).
slowdown            0.01            Time in seconds to wait between objects
==================  ==============  ==========================================

[object-auditor]

==================  ==============  ==========================================
Option              Default         Description
------------------  --------------  ------------------------------------------
log_name            object-auditor  Label used when logging
log_facility        LOG_LOCAL0      Syslog log facility
log_level           INFO            Logging level
log_time            3600            Frequency of status logs in seconds.
files_per_second    20              Maximum files audited per second. Should
                                    be tuned according to individual system
                                    specs. 0 is unlimited.
bytes_per_second    10000000        Maximum bytes audited per second. Should
                                    be tuned according to individual system
                                    specs. 0 is unlimited.
==================  ==============  ==========================================

------------------------------
Container Server Configuration
------------------------------

An example Container Server configuration can be found at
etc/container-server.conf-sample in the source code repository.

The following configuration options are available:

[DEFAULT]

===================  ==========  ============================================
Option               Default     Description
-------------------  ----------  --------------------------------------------
swift_dir            /etc/swift  Swift configuration directory
devices              /srv/node   Parent directory of where devices are mounted
mount_check          true        Whether or not check if the devices are
                                 mounted to prevent accidentally writing
                                 to the root device
bind_ip              0.0.0.0     IP Address for server to bind to
bind_port            6001        Port for server to bind to
bind_timeout         30          Seconds to attempt bind before giving up
workers              auto        Override the number of pre-forked workers
                                 that will accept connections.  If set it
                                 should be an integer, zero means no fork.  If
                                 unset, it will try to default to the number
                                 of effective cpu cores and fallback to one.
                                 Increasing the number of workers may reduce
                                 the possibility of slow file system
                                 operations in one request from negatively
                                 impacting other requests.  See
                                 :ref:`general-service-tuning`
max_clients          1024        Maximum number of clients one worker can
                                 process simultaneously (it will actually
                                 accept(2) N + 1). Setting this to one (1)
                                 will only handle one request at a time,
                                 without accepting another request
                                 concurrently.
user                 swift       User to run as
disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                 underlying filesystem does not support it.
log_custom_handlers  None        Comma-separated list of functions to call
                                 to setup custom log handlers.
eventlet_debug       false       If true, turn on debug logging for eventlet
fallocate_reserve    0           You can set fallocate_reserve to the number of
                                 bytes you'd like fallocate to reserve, whether
                                 there is space for the given file size or not.
                                 This is useful for systems that behave badly
                                 when they completely run out of space; you can
                                 make the services pretend they're out of space
                                 early.
===================  ==========  ============================================

[container-server]

==================  ================  ========================================
Option              Default           Description
------------------  ----------------  ----------------------------------------
use                                   paste.deploy entry point for the
                                      container server.  For most cases, this
                                      should be `egg:swift#container`.
set log_name        container-server  Label used when logging
set log_facility    LOG_LOCAL0        Syslog log facility
set log_level       INFO              Logging level
node_timeout        3                 Request timeout to external services
conn_timeout        0.5               Connection timeout to external services
allow_versions      false             Enable/Disable object versioning feature
==================  ================  ========================================

[container-replicator]

==================  ====================  ====================================
Option              Default               Description
------------------  --------------------  ------------------------------------
log_name            container-replicator  Label used when logging
log_facility        LOG_LOCAL0            Syslog log facility
log_level           INFO                  Logging level
per_diff            1000
concurrency         8                     Number of replication workers to
                                          spawn
run_pause           30                    Time in seconds to wait between
                                          replication passes
node_timeout        10                    Request timeout to external services
conn_timeout        0.5                   Connection timeout to external
                                          services
reclaim_age         604800                Time elapsed in seconds before a
                                          container can be reclaimed
==================  ====================  ====================================

[container-updater]

========================  =================  ==================================
Option                    Default            Description
------------------------  -----------------  ----------------------------------
log_name                  container-updater  Label used when logging
log_facility              LOG_LOCAL0         Syslog log facility
log_level                 INFO               Logging level
interval                  300                Minimum time for a pass to take
concurrency               4                  Number of updater workers to spawn
node_timeout              3                  Request timeout to external
                                             services
conn_timeout              0.5                Connection timeout to external
                                             services
slowdown                  0.01               Time in seconds to wait between
                                             containers
account_suppression_time  60                 Seconds to suppress updating an
                                             account that has generated an
                                             error (timeout, not yet found,
                                             etc.)
========================  =================  ==================================

[container-auditor]

=====================  =================  =======================================
Option                 Default            Description
---------------------  -----------------  ---------------------------------------
log_name               container-auditor  Label used when logging
log_facility           LOG_LOCAL0         Syslog log facility
log_level              INFO               Logging level
interval               1800               Minimum time for a pass to take
containers_per_second  200                Maximum containers audited per second.
                                          Should be tuned according to individual
                                          system specs. 0 is unlimited.
=====================  =================  =======================================

----------------------------
Account Server Configuration
----------------------------

An example Account Server configuration can be found at
etc/account-server.conf-sample in the source code repository.

The following configuration options are available:

[DEFAULT]

===================  ==========  =============================================
Option               Default     Description
-------------------  ----------  ---------------------------------------------
swift_dir            /etc/swift  Swift configuration directory
devices              /srv/node   Parent directory or where devices are mounted
mount_check          true        Whether or not check if the devices are
                                 mounted to prevent accidentally writing
                                 to the root device
bind_ip              0.0.0.0     IP Address for server to bind to
bind_port            6002        Port for server to bind to
bind_timeout         30          Seconds to attempt bind before giving up
workers              auto        Override the number of pre-forked workers
                                 that will accept connections.  If set it
                                 should be an integer, zero means no fork.  If
                                 unset, it will try to default to the number
                                 of effective cpu cores and fallback to one.
                                 Increasing the number of workers may reduce
                                 the possibility of slow file system
                                 operations in one request from negatively
                                 impacting other requests.  See
                                 :ref:`general-service-tuning`
max_clients          1024        Maximum number of clients one worker can
                                 process simultaneously (it will actually
                                 accept(2) N + 1). Setting this to one (1)
                                 will only handle one request at a time,
                                 without accepting another request
                                 concurrently.
user                 swift       User to run as
db_preallocation     off         If you don't mind the extra disk space usage in
                                 overhead, you can turn this on to preallocate
                                 disk space with SQLite databases to decrease
                                 fragmentation.
disable_fallocate    false       Disable "fast fail" fallocate checks if the
                                 underlying filesystem does not support it.
log_custom_handlers  None        Comma-separated list of functions to call
                                 to setup custom log handlers.
eventlet_debug       false       If true, turn on debug logging for eventlet
fallocate_reserve    0           You can set fallocate_reserve to the number of
                                 bytes you'd like fallocate to reserve, whether
                                 there is space for the given file size or not.
                                 This is useful for systems that behave badly
                                 when they completely run out of space; you can
                                 make the services pretend they're out of space
                                 early.
===================  ==========  =============================================

[account-server]

==================  ==============  ==========================================
Option              Default         Description
------------------  --------------  ------------------------------------------
use                                 Entry point for paste.deploy for the account
                                    server.  For most cases, this should be
                                    `egg:swift#account`.
set log_name        account-server  Label used when logging
set log_facility    LOG_LOCAL0      Syslog log facility
set log_level       INFO            Logging level
==================  ==============  ==========================================

[account-replicator]

==================  ==================  ======================================
Option              Default             Description
------------------  ------------------  --------------------------------------
log_name            account-replicator  Label used when logging
log_facility        LOG_LOCAL0          Syslog log facility
log_level           INFO                Logging level
per_diff            1000
concurrency         8                   Number of replication workers to spawn
run_pause           30                  Time in seconds to wait between
                                        replication passes
node_timeout        10                  Request timeout to external services
conn_timeout        0.5                 Connection timeout to external services
reclaim_age         604800              Time elapsed in seconds before an
                                        account can be reclaimed
==================  ==================  ======================================

[account-auditor]

====================  ===============  =======================================
Option                Default          Description
--------------------  ---------------  ---------------------------------------
log_name              account-auditor  Label used when logging
log_facility          LOG_LOCAL0       Syslog log facility
log_level             INFO             Logging level
interval              1800             Minimum time for a pass to take
accounts_per_second   200              Maximum accounts audited per second.
                                       Should be tuned according to individual
                                       system specs. 0 is unlimited.
====================  ===============  =======================================

[account-reaper]

==================  ===============  =========================================
Option              Default          Description
------------------  ---------------  -----------------------------------------
log_name            account-auditor  Label used when logging
log_facility        LOG_LOCAL0       Syslog log facility
log_level           INFO             Logging level
concurrency         25               Number of replication workers to spawn
interval            3600             Minimum time for a pass to take
node_timeout        10               Request timeout to external services
conn_timeout        0.5              Connection timeout to external services
delay_reaping       0                Normally, the reaper begins deleting
                                     account information for deleted accounts
                                     immediately; you can set this to delay
                                     its work however. The value is in seconds,
                                     2592000 = 30 days, for example.
==================  ===============  =========================================

.. _proxy-server-config:

--------------------------
Proxy Server Configuration
--------------------------

An example Proxy Server configuration can be found at
etc/proxy-server.conf-sample in the source code repository.

The following configuration options are available:

[DEFAULT]

============================  ===============  =============================
Option                        Default          Description
----------------------------  ---------------  -----------------------------
bind_ip                       0.0.0.0          IP Address for server to
                                               bind to
bind_port                     80               Port for server to bind to
bind_timeout                  30               Seconds to attempt bind before
                                               giving up
swift_dir                     /etc/swift       Swift configuration directory
workers                       auto             Override the number of
                                               pre-forked workers that will
                                               accept connections.  If set it
                                               should be an integer, zero
                                               means no fork.  If unset, it
                                               will try to default to the
                                               number of effective cpu cores
                                               and fallback to one.  See
                                               :ref:`general-service-tuning`
max_clients                   1024             Maximum number of clients one
                                               worker can process
                                               simultaneously (it will
                                               actually accept(2) N +
                                               1). Setting this to one (1)
                                               will only handle one request at
                                               a time, without accepting
                                               another request
                                               concurrently.
user                          swift            User to run as
cert_file                                      Path to the ssl .crt. This
                                               should be enabled for testing
                                               purposes only.
key_file                                       Path to the ssl .key. This
                                               should be enabled for testing
                                               purposes only.
cors_allow_origin                              This is a list of hosts that
                                               are included with any CORS
                                               request by default and
                                               returned with the
                                               Access-Control-Allow-Origin
                                               header in addition to what
                                               the container has set.
log_custom_handlers           None             Comma separated list of functions
                                               to call to setup custom log
                                               handlers.
eventlet_debug                false            If true, turn on debug logging
                                               for eventlet

expose_info                   true             Enables exposing configuration
                                               settings via HTTP GET /info.

admin_key                                      Key to use for admin calls that
                                               are HMAC signed.  Default
                                               is empty, which will
                                               disable admin calls to
                                               /info.
============================  ===============  =============================

[proxy-server]

============================  ===============  =============================
Option                        Default          Description
----------------------------  ---------------  -----------------------------
use                                            Entry point for paste.deploy for
                                               the proxy server.  For most
                                               cases, this should be
                                               `egg:swift#proxy`.
set log_name                  proxy-server     Label used when logging
set log_facility              LOG_LOCAL0       Syslog log facility
set log_level                 INFO             Log level
set log_headers               True             If True, log headers in each
                                               request
set log_handoffs              True             If True, the proxy will log
                                               whenever it has to failover to a
                                               handoff node
recheck_account_existence     60               Cache timeout in seconds to
                                               send memcached for account
                                               existence
recheck_container_existence   60               Cache timeout in seconds to
                                               send memcached for container
                                               existence
object_chunk_size             65536            Chunk size to read from
                                               object servers
client_chunk_size             65536            Chunk size to read from
                                               clients
memcache_servers              127.0.0.1:11211  Comma separated list of
                                               memcached servers ip:port
memcache_max_connections      2                Max number of connections to
                                               each memcached server per
                                               worker
node_timeout                  10               Request timeout to external
                                               services
recoverable_node_timeout      node_timeout     Request timeout to external
                                               services for requests that, on
                                               failure, can be recovered
                                               from. For example, object GET.
client_timeout                60               Timeout to read one chunk
                                               from a client
conn_timeout                  0.5              Connection timeout to
                                               external services
error_suppression_interval    60               Time in seconds that must
                                               elapse since the last error
                                               for a node to be considered
                                               no longer error limited
error_suppression_limit       10               Error count to consider a
                                               node error limited
allow_account_management      false            Whether account PUTs and DELETEs
                                               are even callable
object_post_as_copy           true             Set object_post_as_copy = false
                                               to turn on fast posts where only
                                               the metadata changes are stored
                                               anew and the original data file
                                               is kept in place. This makes for
                                               quicker posts; but since the
                                               container metadata isn't updated
                                               in this mode, features like
                                               container sync won't be able to
                                               sync posts.
account_autocreate            false            If set to 'true' authorized
                                               accounts that do not yet exist
                                               within the Swift cluster will
                                               be automatically created.
max_containers_per_account    0                If set to a positive value,
                                               trying to create a container
                                               when the account already has at
                                               least this maximum containers
                                               will result in a 403 Forbidden.
                                               Note: This is a soft limit,
                                               meaning a user might exceed the
                                               cap for
                                               recheck_account_existence before
                                               the 403s kick in.
max_containers_whitelist                       This is a comma separated list
                                               of account names that ignore
                                               the max_containers_per_account
                                               cap.
rate_limit_after_segment      10               Rate limit the download of
                                               large object segments after
                                               this segment is downloaded.
rate_limit_segments_per_sec   1                Rate limit large object
                                               downloads at this rate.
request_node_count            2 * replicas     Set to the number of nodes to
                                               contact for a normal request.
                                               You can use '* replicas' at the
                                               end to have it use the number
                                               given times the number of
                                               replicas for the ring being used
                                               for the request.
swift_owner_headers                    up to the auth system in use,
                                               but usually indicates
                                               administrative responsibilities.
============================  ===============  =============================

[tempauth]

=====================  =============================== =======================
Option                 Default                         Description
---------------------  ------------------------------- -----------------------
use                                                    Entry point for
                                                       paste.deploy to use for
                                                       auth. To use tempauth
                                                       set to:
                                                       `egg:swift#tempauth`
set log_name           tempauth                        Label used when logging
set log_facility       LOG_LOCAL0                      Syslog log facility
set log_level          INFO                            Log level
set log_headers        True                            If True, log headers in
                                                       each request
reseller_prefix        AUTH                            The naming scope for the
                                                       auth service. Swift
                                                       storage accounts and
                                                       auth tokens will begin
                                                       with this prefix.
auth_prefix            /auth/                          The HTTP request path
                                                       prefix for the auth
                                                       service. Swift itself
                                                       reserves anything
                                                       beginning with the
                                                       letter `v`.
token_life             86400                           The number of seconds a
                                                       token is valid.
storage_url_scheme     default                         Scheme to return with
                                                       storage urls: http,
                                                       https, or default
                                                       (chooses based on what
                                                       the server is running
                                                       as) This can be useful
                                                       with an SSL load
                                                       balancer in front of a
                                                       non-SSL server.
=====================  =============================== =======================

Additionally, you need to list all the accounts/users you want here. The format
is::

    user__ =  [group] [group] [...] [storage_url]

or if you want to be able to include underscores in the ```` or
```` portions, you can base64 encode them (with *no* equal signs) in a
line like this::

    user64__ =  [group] [group] [...] [storage_url]

There are special groups of::

    .reseller_admin = can do anything to any account for this auth
    .admin = can do anything within the account

If neither of these groups are specified, the user can only access containers
that have been explicitly allowed for them by a .admin or .reseller_admin.

The trailing optional storage_url allows you to specify an alternate url to
hand back to the user upon authentication. If not specified, this defaults to::

    $HOST/v1/_

Where $HOST will do its best to resolve to what the requester would need to use
to reach this host,  is from this section, and  is
from the user__ name. Note that $HOST cannot possibly handle
when you have a load balancer in front of it that does https while TempAuth
itself runs with http; in such a case, you'll have to specify the
storage_url_scheme configuration value as an override.

Here are example entries, required for running the tests::

    user_admin_admin = admin .admin .reseller_admin
    user_test_tester = testing .admin
    user_test2_tester2 = testing2 .admin
    user_test_tester3 = testing3

    # account "test_y" and user "tester_y" (note the lack of padding = chars)
    user64_dGVzdF95_dGVzdGVyX3k = testing4 .admin

------------------------
Memcached Considerations
------------------------

Several of the Services rely on Memcached for caching certain types of
lookups, such as auth tokens, and container/account existence.  Swift does
not do any caching of actual object data.  Memcached should be able to run
on any servers that have available RAM and CPU.  At Rackspace, we run
Memcached on the proxy servers.  The `memcache_servers` config option
in the `proxy-server.conf` should contain all memcached servers.

-----------
System Time
-----------

Time may be relative but it is relatively important for Swift!  Swift uses
timestamps to determine which is the most recent version of an object.
It is very important for the system time on each server in the cluster to
by synced as closely as possible (more so for the proxy server, but in general
it is a good idea for all the servers).  At Rackspace, we use NTP with a local
NTP server to ensure that the system times are as close as possible.  This
should also be monitored to ensure that the times do not vary too much.

.. _general-service-tuning:

----------------------
General Service Tuning
----------------------

Most services support either a `worker` or `concurrency` value in the
settings.  This allows the services to make effective use of the cores
available. A good starting point to set the concurrency level for the proxy
and storage services to 2 times the number of cores available. If more than
one service is sharing a server, then some experimentation may be needed to
find the best balance.

At Rackspace, our Proxy servers have dual quad core processors, giving us 8
cores. Our testing has shown 16 workers to be a pretty good balance when
saturating a 10g network and gives good CPU utilization.

Our Storage servers all run together on the same servers. These servers have
dual quad core processors, for 8 cores total. We run the Account, Container,
and Object servers with 8 workers each. Most of the background jobs are run at
a concurrency of 1, with the exception of the replicators which are run at a
concurrency of 2.

The `max_clients` parameter can be used to adjust the number of client
requests an individual worker accepts for processing. The fewer requests being
processed at one time, the less likely a request that consumes the worker's
CPU time, or blocks in the OS, will negatively impact other requests. The more
requests being processed at one time, the more likely one worker can utilize
network and disk capacity.

On systems that have more cores, and more memory, where one can afford to run
more workers, raising the number of workers and lowering the maximum number of
clients serviced per worker can lessen the impact of CPU intensive or stalled
requests.

The above configuration setting should be taken as suggestions and testing
of configuration settings should be done to ensure best utilization of CPU,
network connectivity, and disk I/O.

-------------------------
Filesystem Considerations
-------------------------

Swift is designed to be mostly filesystem agnostic--the only requirement
being that the filesystem supports extended attributes (xattrs). After
thorough testing with our use cases and hardware configurations, XFS was
the best all-around choice. If you decide to use a filesystem other than
XFS, we highly recommend thorough testing.

For distros with more recent kernels (for example Ubuntu 12.04 Precise),
we recommend using the default settings (including the default inode size
of 256 bytes) when creating the file system::

    mkfs.xfs /dev/sda1

In the last couple of years, XFS has made great improvements in how inodes
are allocated and used.  Using the default inode size no longer has an
impact on performance.

For distros with older kernels (for example Ubuntu 10.04 Lucid),
some settings can dramatically impact performance. We recommend the
following when creating the file system::

    mkfs.xfs -i size=1024 /dev/sda1

Setting the inode size is important, as XFS stores xattr data in the inode.
If the metadata is too large to fit in the inode, a new extent is created,
which can cause quite a performance problem. Upping the inode size to 1024
bytes provides enough room to write the default metadata, plus a little
headroom.

The following example mount options are recommended when using XFS::

    mount -t xfs -o noatime,nodiratime,nobarrier,logbufs=8 /dev/sda1 /srv/node/sda

We do not recommend running Swift on RAID, but if you are using
RAID it is also important to make sure that the proper sunit and swidth
settings get set so that XFS can make most efficient use of the RAID array.

For a standard swift install, all data drives are mounted directly under
/srv/node (as can be seen in the above example of mounting /def/sda1 as
/srv/node/sda). If you choose to mount the drives in another directory,
be sure to set the `devices` config option in all of the server configs to
point to the correct directory.

Swift uses system calls to reserve space for new objects being written into
the system. If your filesystem does not support `fallocate()` or
`posix_fallocate()`, be sure to set the `disable_fallocate = true` config
parameter in account, container, and object server configs.

---------------------
General System Tuning
---------------------

Rackspace currently runs Swift on Ubuntu Server 10.04, and the following
changes have been found to be useful for our use cases.

The following settings should be in `/etc/sysctl.conf`::

    # disable TIME_WAIT.. wait..
    net.ipv4.tcp_tw_recycle=1
    net.ipv4.tcp_tw_reuse=1

    # disable syn cookies
    net.ipv4.tcp_syncookies = 0

    # double amount of allowed conntrack
    net.ipv4.netfilter.ip_conntrack_max = 262144

To load the updated sysctl settings, run ``sudo sysctl -p``

A note about changing the TIME_WAIT values.  By default the OS will hold
a port open for 60 seconds to ensure that any remaining packets can be
received.  During high usage, and with the number of connections that are
created, it is easy to run out of ports.  We can change this since we are
in control of the network.  If you are not in control of the network, or
do not expect high loads, then you may not want to adjust those values.

----------------------
Logging Considerations
----------------------

Swift is set up to log directly to syslog. Every service can be configured
with the `log_facility` option to set the syslog log facility destination.
We recommended using syslog-ng to route the logs to specific log
files locally on the server and also to remote log collecting servers.
Additionally, custom log handlers can be used via the custom_log_handlers
setting.
swift-1.13.1/doc/source/overview_container_sync.rst0000664000175400017540000004222212323703611023632 0ustar  jenkinsjenkins00000000000000======================================
Container to Container Synchronization
======================================

--------
Overview
--------

Swift has a feature where all the contents of a container can be mirrored to
another container through background synchronization. Swift cluster operators
configure their cluster to allow/accept sync requests to/from other clusters,
and the user specifies where to sync their container to along with a secret
synchronization key.

.. note::

    Container sync will sync object POSTs only if the proxy server is set to
    use "object_post_as_copy = true" which is the default. So-called fast
    object posts, "object_post_as_copy = false" do not update the container
    listings and therefore can't be detected for synchronization.

.. note::

    If you are using the large objects feature you will need to ensure both
    your manifest file and your segment files are synced if they happen to be
    in different containers.

--------------------------
Configuring Container Sync
--------------------------

Create a container-sync-realms.conf file specifying the allowable clusters
and their information::

    [realm1]
    key = realm1key
    key2 = realm1key2
    cluster_name1 = https://host1/v1/
    cluster_name2 = https://host2/v1/

    [realm2]
    key = realm2key
    key2 = realm2key2
    cluster_name3 = https://host3/v1/
    cluster_name4 = https://host4/v1/


Each section name is the name of a sync realm. A sync realm is a set of
clusters that have agreed to allow container syncing with each other. Realm
names will be considered case insensitive.

The key is the overall cluster-to-cluster key used in combination with the
external users' key that they set on their containers' X-Container-Sync-Key
metadata header values. These keys will be used to sign each request the
container sync daemon makes and used to validate each incoming container sync
request.

The key2 is optional and is an additional key incoming requests will be checked
against. This is so you can rotate keys if you wish; you move the existing key
to key2 and make a new key value.

Any values in the realm section whose names begin with cluster\_ will indicate
the name and endpoint of a cluster and will be used by external users in
their containers' X-Container-Sync-To metadata header values with the format
"//realm_name/cluster_name/account_name/container_name". Realm and cluster
names are considered case insensitive.

The endpoint is what the container sync daemon will use when sending out
requests to that cluster. Keep in mind this endpoint must be reachable by all
container servers, since that is where the container sync daemon runs. Note
that the endpoint ends with /v1/ and that the container sync daemon will then
add the account/container/obj name after that.

Distribute this container-sync-realms.conf file to all your proxy servers
and container servers.

You also need to add the container_sync middleware to your proxy pipeline. It
needs to be after any memcache middleware and before any auth middleware. The
container_sync section only needs the "use" item. For example::

    [pipeline:main]
    pipeline = healthcheck proxy-logging cache container_sync tempauth proxy-logging proxy-server

    [filter:container_sync]
    use = egg:swift#container_sync


-------------------------------------------------------
Old-Style: Configuring a Cluster's Allowable Sync Hosts
-------------------------------------------------------

This section is for the old-style of using container sync. See the previous
section, Configuring Container Sync, for the new-style.

With the old-style, the Swift cluster operator must allow synchronization with
a set of hosts before the user can enable container synchronization. First, the
backend container server needs to be given this list of hosts in the
container-server.conf file::

    [DEFAULT]
    # This is a comma separated list of hosts allowed in the
    # X-Container-Sync-To field for containers.
    # allowed_sync_hosts = 127.0.0.1
    allowed_sync_hosts = host1,host2,etc.
    ...

    [container-sync]
    # You can override the default log routing for this app here (don't
    # use set!):
    # log_name = container-sync
    # log_facility = LOG_LOCAL0
    # log_level = INFO
    # Will sync, at most, each container once per interval
    # interval = 300
    # Maximum amount of time to spend syncing each container
    # container_time = 60


----------------------
Logging Container Sync
----------------------

Tracking sync progress, problems, and just general activity can only be
achieved with log processing currently for container synchronization. In that
light, you may wish to set the above `log_` options to direct the
container-sync logs to a different file for easier monitoring. Additionally, it
should be noted there is no way for an end user to detect sync progress or
problems other than HEADing both containers and comparing the overall
information.

----------------------------------------------------------
Using the ``swift`` tool to set up synchronized containers
----------------------------------------------------------

.. note::

    The ``swift`` tool is available from the `python-swiftclient`_ library.

.. note::

    You must be the account admin on the account to set synchronization targets
    and keys.

You simply tell each container where to sync to and give it a secret
synchronization key. First, let's get the account details for our two cluster
accounts::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v
    StorageURL: http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e
    Auth Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19
       Account: AUTH_208d1854-e475-4500-b315-81de645d060e
    Containers: 0
       Objects: 0
         Bytes: 0

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v
    StorageURL: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c
    Auth Token: AUTH_tk816a1aaf403c49adb92ecfca2f88e430
       Account: AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c
    Containers: 0
       Objects: 0
         Bytes: 0

Now, let's make our first container and tell it to synchronize to a second
we'll make next::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing post \
      -t '//realm_name/cluster2_name/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \
      -k 'secret' container1

The ``-t`` indicates the cluster to sync to, which is the realm name of the
section from container-sync-realms.conf, followed by the cluster name from
that section, followed by the account and container names we want to sync to.
The ``-k`` specifies the secret key the two containers will share for
synchronization; this is the user key, the cluster key in
container-sync-realms.conf will also be used behind the scenes.

Now, we'll do something similar for the second cluster's container::

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \
      -t '//realm_name/cluster1_name/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' \
      -k 'secret' container2

That's it. Now we can upload a bunch of stuff to the first container and watch
as it gets synchronized over to the second::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing \
      upload container1 .
    photo002.png
    photo004.png
    photo001.png
    photo003.png

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \
      list container2

    [Nothing there yet, so we wait a bit...]
    [If you're an operator running SAIO and just testing, you may need to
     run 'swift-init container-sync once' to perform a sync scan.]

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \
      list container2
    photo001.png
    photo002.png
    photo003.png
    photo004.png

You can also set up a chain of synced containers if you want more than two.
You'd point 1 -> 2, then 2 -> 3, and finally 3 -> 1 for three containers.
They'd all need to share the same secret synchronization key.

.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient

-----------------------------------
Using curl (or other tools) instead
-----------------------------------

So what's ``swift`` doing behind the scenes? Nothing overly complicated. It
translates the ``-t `` option into an ``X-Container-Sync-To: ``
header and the ``-k `` option into an ``X-Container-Sync-Key: ``
header.

For instance, when we created the first container above and told it to
synchronize to the second, we could have used this curl command::

    $ curl -i -X POST -H 'X-Auth-Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19' \
      -H 'X-Container-Sync-To: //realm_name/cluster2_name/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \
      -H 'X-Container-Sync-Key: secret' \
      'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1'
    HTTP/1.1 204 No Content
    Content-Length: 0
    Content-Type: text/plain; charset=UTF-8
    Date: Thu, 24 Feb 2011 22:39:14 GMT

---------------------------------------------------------------------
Old-Style: Using the ``swift`` tool to set up synchronized containers
---------------------------------------------------------------------

.. note::

    The ``swift`` tool is available from the `python-swiftclient`_ library.

.. note::

    You must be the account admin on the account to set synchronization targets
    and keys.

This is for the old-style of container syncing using allowed_sync_hosts.

You simply tell each container where to sync to and give it a secret
synchronization key. First, let's get the account details for our two cluster
accounts::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing stat -v
    StorageURL: http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e
    Auth Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19
       Account: AUTH_208d1854-e475-4500-b315-81de645d060e
    Containers: 0
       Objects: 0
         Bytes: 0

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 stat -v
    StorageURL: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c
    Auth Token: AUTH_tk816a1aaf403c49adb92ecfca2f88e430
       Account: AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c
    Containers: 0
       Objects: 0
         Bytes: 0

Now, let's make our first container and tell it to synchronize to a second
we'll make next::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing post \
      -t 'http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \
      -k 'secret' container1

The ``-t`` indicates the URL to sync to, which is the ``StorageURL`` from
cluster2 we retrieved above plus the container name. The ``-k`` specifies the
secret key the two containers will share for synchronization. Now, we'll do
something similar for the second cluster's container::

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 post \
      -t 'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1' \
      -k 'secret' container2

That's it. Now we can upload a bunch of stuff to the first container and watch
as it gets synchronized over to the second::

    $ swift -A http://cluster1/auth/v1.0 -U test:tester -K testing \
      upload container1 .
    photo002.png
    photo004.png
    photo001.png
    photo003.png

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \
      list container2

    [Nothing there yet, so we wait a bit...]
    [If you're an operator running SAIO and just testing, you may need to
     run 'swift-init container-sync once' to perform a sync scan.]

    $ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \
      list container2
    photo001.png
    photo002.png
    photo003.png
    photo004.png

You can also set up a chain of synced containers if you want more than two.
You'd point 1 -> 2, then 2 -> 3, and finally 3 -> 1 for three containers.
They'd all need to share the same secret synchronization key.

.. _`python-swiftclient`: http://github.com/openstack/python-swiftclient

----------------------------------------------
Old-Style: Using curl (or other tools) instead
----------------------------------------------

This is for the old-style of container syncing using allowed_sync_hosts.

So what's ``swift`` doing behind the scenes? Nothing overly complicated. It
translates the ``-t `` option into an ``X-Container-Sync-To: ``
header and the ``-k `` option into an ``X-Container-Sync-Key: ``
header.

For instance, when we created the first container above and told it to
synchronize to the second, we could have used this curl command::

    $ curl -i -X POST -H 'X-Auth-Token: AUTH_tkd5359e46ff9e419fa193dbd367f3cd19' \
      -H 'X-Container-Sync-To: http://cluster2/v1/AUTH_33cdcad8-09fb-4940-90da-0f00cbf21c7c/container2' \
      -H 'X-Container-Sync-Key: secret' \
      'http://cluster1/v1/AUTH_208d1854-e475-4500-b315-81de645d060e/container1'
    HTTP/1.1 204 No Content
    Content-Length: 0
    Content-Type: text/plain; charset=UTF-8
    Date: Thu, 24 Feb 2011 22:39:14 GMT

--------------------------------------------------
What's going on behind the scenes, in the cluster?
--------------------------------------------------

The swift-container-sync does the job of sending updates to the remote
container.

This is done by scanning the local devices for container databases and
checking for x-container-sync-to and x-container-sync-key metadata values.
If they exist, newer rows since the last sync will trigger PUTs or DELETEs
to the other container.

.. note::

    The swift-container-sync process runs on each container server in the
    cluster and talks to the proxy servers (or load balancers) in the remote
    cluster. Therefore, the container servers must be permitted to initiate
    outbound connections to the remote proxy servers (or load balancers).

.. note::

    Container sync will sync object POSTs only if the proxy server is set to
    use "object_post_as_copy = true" which is the default. So-called fast
    object posts, "object_post_as_copy = false" do not update the container
    listings and therefore can't be detected for synchronization.

The actual syncing is slightly more complicated to make use of the three
(or number-of-replicas) main nodes for a container without each trying to
do the exact same work but also without missing work if one node happens to
be down.

Two sync points are kept in each container database. When syncing a
container, the container-sync process figures out which replica of the
container it has. In a standard 3-replica scenario, the process will
have either replica number 0, 1, or 2. This is used to figure out
which rows are belong to this sync process and which ones don't.

An example may help. Assume a replica count of 3 and database row IDs
are 1..6. Also, assume that container-sync is running on this
container for the first time, hence SP1 = SP2 = -1. ::

   SP1
   SP2
    |
    v
   -1 0 1 2 3 4 5 6

First, the container-sync process looks for rows with id between SP1
and SP2. Since this is the first run, SP1 = SP2 = -1, and there aren't
any such rows. ::

   SP1
   SP2
    |
    v
   -1 0 1 2 3 4 5 6

Second, the container-sync process looks for rows with id greater than
SP1, and syncs those rows which it owns. Ownership is based on the
hash of the object name, so it's not always guaranteed to be exactly
one out of every three rows, but it usually gets close. For the sake
of example, let's say that this process ends up owning rows 2 and 5.

Once it's finished trying to sync those rows, it updates SP1 to be the
biggest row-id that it's seen, which is 6 in this example. ::

   SP2           SP1
    |             |
    v             v
   -1 0 1 2 3 4 5 6

While all that was going on, clients uploaded new objects into the
container, creating new rows in the database. ::

   SP2           SP1
    |             |
    v             v
   -1 0 1 2 3 4 5 6 7 8 9 10 11 12

On the next run, the container-sync starts off looking at rows with
ids between SP1 and SP2. This time, there are a bunch of them. The
sync process try to sync all of them. If it succeeds, it will set
SP2 to equal SP1. If it fails, it will set SP2 to the failed object
and will continue to try all other objects till SP1, setting SP2 to
the first object that failed.

Under normal circumstances, the container-sync processes
will have already taken care of synchronizing all rows, between SP1
and SP2, resulting in a set of quick checks.
However, if one of the sync
processes failed for some reason, then this is a vital fallback to
make sure all the objects in the container get synchronized. Without
this seemingly-redundant work, any container-sync failure results in
unsynchronized objects. Note that the container sync will persistently
retry to sync any faulty object until success, while logging each failure.

Once it's done with the fallback rows, and assuming no faults occurred,
SP2 is advanced to SP1. ::

                 SP2
                 SP1
                  |
                  v
   -1 0 1 2 3 4 5 6 7 8 9 10 11 12

Then, rows with row ID greater than SP1 are synchronized (provided
this container-sync process is responsible for them), and SP1 is moved
up to the greatest row ID seen. ::

                 SP2            SP1
                  |              |
                  v              v
   -1 0 1 2 3 4 5 6 7 8 9 10 11 12
swift-1.13.1/doc/source/overview_reaper.rst0000664000175400017540000001100112323703611022061 0ustar  jenkinsjenkins00000000000000==================
The Account Reaper
==================

The Account Reaper removes data from deleted accounts in the background.

An account is marked for deletion by a reseller issuing a DELETE request on the
account's storage URL. This simply puts the value DELETED into the status
column of the account_stat table in the account database (and replicas),
indicating the data for the account should be deleted later.

There is normally no set retention time and no undelete; it is assumed the
reseller will implement such features and only call DELETE on the account once
it is truly desired the account's data be removed. However, in order to protect
the Swift cluster accounts from an improper or mistaken delete request, you can
set a delay_reaping value in the [account-reaper] section of the
account-server.conf to delay the actual deletion of data. At this time, there
is no utility to undelete an account; one would have to update the account
database replicas directly, setting the status column to an empty string and
updating the put_timestamp to be greater than the delete_timestamp. (On the
TODO list is writing a utility to perform this task, preferably through a ReST
call.)

The account reaper runs on each account server and scans the server
occasionally for account databases marked for deletion. It will only trigger on
accounts that server is the primary node for, so that multiple account servers
aren't all trying to do the same work at the same time. Using multiple servers
to delete one account might improve deletion speed, but requires coordination
so they aren't duplicating effort. Speed really isn't as much of a concern with
data deletion and large accounts aren't deleted that often.

The deletion process for an account itself is pretty straightforward. For each
container in the account, each object is deleted and then the container is
deleted. Any deletion requests that fail won't stop the overall process, but
will cause the overall process to fail eventually (for example, if an object
delete times out, the container won't be able to be deleted later and therefore
the account won't be deleted either). The overall process continues even on a
failure so that it doesn't get hung up reclaiming cluster space because of one
troublesome spot. The account reaper will keep trying to delete an account
until it eventually becomes empty, at which point the database reclaim process
within the db_replicator will eventually remove the database files.

Sometimes a persistent error state can prevent some object or container
from being deleted. If this happens, you will see a message such as "Account
 has not been reaped since " in the log. You can control when
this is logged with the reap_warn_after value in the [account-reaper] section
of the account-server.conf file. By default this is 30 days.

-------
History
-------

At first, a simple approach of deleting an account through completely external
calls was considered as it required no changes to the system. All data would
simply be deleted in the same way the actual user would, through the public
ReST API. However, the downside was that it would use proxy resources and log
everything when it didn't really need to. Also, it would likely need a
dedicated server or two, just for issuing the delete requests.

A completely bottom-up approach was also considered, where the object and
container servers would occasionally scan the data they held and check if the
account was deleted, removing the data if so. The upside was the speed of
reclamation with no impact on the proxies or logging, but the downside was that
nearly 100% of the scanning would result in no action creating a lot of I/O
load for no reason.

A more container server centric approach was also considered, where the account
server would mark all the containers for deletion and the container servers
would delete the objects in each container and then themselves. This has the
benefit of still speedy reclamation for accounts with a lot of containers, but
has the downside of a pretty big load spike. The process could be slowed down
to alleviate the load spike possibility, but then the benefit of speedy
reclamation is lost and what's left is just a more complex process. Also,
scanning all the containers for those marked for deletion when the majority
wouldn't be seemed wasteful. The db_replicator could do this work while
performing its replication scan, but it would have to spawn and track deletion
processes which seemed needlessly complex.

In the end, an account server centric approach seemed best, as described above.
swift-1.13.1/doc/source/apache_deployment_guide.rst0000664000175400017540000001500712323703611023525 0ustar  jenkinsjenkins00000000000000=======================
Apache Deployment Guide
=======================

----------------------------
Web Front End Considerations
----------------------------

Swift can be configured to work both using an integral web front-end
and using a full-fledged Web Server such as the Apache2 (HTTPD) web server.
The integral web front-end is a wsgi mini "Web Server" which opens
up its own socket and serves http requests directly.
The incoming requests accepted by the integral web front-end are then forwarded
to a wsgi application (the core swift) for further handling, possibly
via wsgi middleware sub-components.

client<---->'integral web front-end'<---->middleware<---->'core swift'

To gain full advantage of Apache2, Swift can alternatively be
configured to work as a request processor of the Apache2 server.
This alternative deployment scenario uses mod_wsgi of Apache2
to forward requests to the swift wsgi application and middleware.

client<---->'Apache2 with mod_wsgi'<----->middleware<---->'core swift'

The integral web front-end offers simplicity and requires
minimal configuration. It is also the web front-end most commonly used
with Swift.
Additionlly, the integral web front-end includes support for
receiving chunked transfer encoding from a client,
presently not supported by Apache2 in the operation mode described here.

The use of Apache2 offers new ways to extend Swift and integrate it with
existing authentication, administration and control systems.
A single Apache2 server can serve as the web front end of any number of swift
servers residing on a swift node.
For example when a storage node offers account, container and object services,
a single Apache2 server can serve as the web front end of all three services.

The apache variant described here was tested as part of an IBM research work.
It was found that following tuning, the Apache2 offer generally equivalent
performance to that offered by the integral web front-end.
Alternative to Apache2, other web servers may be used, but were never tested.

-------------
Apache2 Setup
-------------
Both Apache2 and mod-wsgi needs to be installed on the system.
Ubuntu comes with Apache2 installed. Install mod-wsgi using:: 

    sudo apt-get install libapache2-mod-wsgi

First, change the User and Group IDs of Apache2 to be those used by Swift.
For example in /etc/apache2/envvars use::

    export APACHE_RUN_USER=swift
    export APACHE_RUN_GROUP=swift

Create a directory for the Apache2 wsgi files::

    sudo mkdir /var/www/swift

Create a file for each service under /var/www/swift.

For a proxy service create /var/www/swift/proxy-server.wsgi::

    from swift.common.wsgi import init_request_processor
    application, conf, logger, log_name = \
        init_request_processor('/etc/swift/proxy-server.conf','proxy-server')

For an account service create /var/www/swift/account-server.wsgi::

    from swift.common.wsgi import init_request_processor
    application, conf, logger, log_name = \
        init_request_processor('/etc/swift/account-server.conf',
                               'account-server')

For an container service create /var/www/swift/container-server.wsgi::

    from swift.common.wsgi import init_request_processor
    application, conf, logger, log_name = \
        init_request_processor('/etc/swift/container-server.conf',
                              'container-server')

For an object service create /var/www/swift/object-server.wsgi::

    from swift.common.wsgi import init_request_processor
    application, conf, logger, log_name = \
        init_request_processor('/etc/swift/object-server.conf',
                               'object-server')

Create a /etc/apache2/conf.d/swift_wsgi.conf configuration file that will
define a port and Virtual Host per each local service.
For example an Apache2 serving as a web front end of a proxy service::

    #Proxy
    NameVirtualHost *:8080
    Listen 8080
    
        ServerName proxy-server
        LimitRequestBody 5368709122
        WSGIDaemonProcess proxy-server processes=5 threads=1
        WSGIProcessGroup proxy-server
        WSGIScriptAlias / /var/www/swift/proxy-server.wsgi
        LimitRequestFields 200
        ErrorLog /var/log/apache2/proxy-server
        LogLevel debug
        CustomLog /var/log/apache2/proxy.log combined
    

Notice that when using Apache the limit on the maximal object size should
be imposed by Apache using the LimitRequestBody rather by the swift proxy.
Note also that the LimitRequestBody should indicate the same value
as indicated by max_file_size located in both 
/etc/swift/swift.conf and in /etc/swift/test.conf.
The Swift default value for max_file_size (when not present) is 5368709122.
For example an Apache2 serving as a web front end of a storage node::

    #Object Service
    NameVirtualHost *:6000
    Listen 6000
    
        ServerName object-server
        WSGIDaemonProcess object-server processes=5 threads=1
        WSGIProcessGroup object-server
        WSGIScriptAlias / /var/www/swift/object-server.wsgi
        LimitRequestFields 200
        ErrorLog /var/log/apache2/object-server
        LogLevel debug
        CustomLog /var/log/apache2/access.log combined
    

    #Container Service
    NameVirtualHost *:6001
    Listen 6001
    
        ServerName container-server
        WSGIDaemonProcess container-server processes=5 threads=1
        WSGIProcessGroup container-server
        WSGIScriptAlias / /var/www/swift/container-server.wsgi
        LimitRequestFields 200
        ErrorLog /var/log/apache2/container-server
        LogLevel debug
        CustomLog /var/log/apache2/access.log combined
    

    #Account Service
    NameVirtualHost *:6002
    Listen 6002
    
        ServerName account-server
        WSGIDaemonProcess account-server processes=5 threads=1
        WSGIProcessGroup account-server
        WSGIScriptAlias / /var/www/swift/account-server.wsgi
        LimitRequestFields 200
        ErrorLog /var/log/apache2/account-server
        LogLevel debug
        CustomLog /var/log/apache2/access.log combined
    

Next stop the Apache2 and start it again (apache2ctl restart is not enough)::

    apache2ctl stop
    apache2ctl start

Edit the tests config file and add::

    web_front_end = apache2
    normalized_urls = True

Also check to see that the file includes max_file_size of the same value as
used for the LimitRequestBody in the apache config file above.

We are done.
You may run functional tests to test - e.g.::

    cd ~swift/swift
    ./.functests
swift-1.13.1/doc/source/conf.py0000664000175400017540000001641412323703611017437 0ustar  jenkinsjenkins00000000000000# -*- coding: utf-8 -*-
# Copyright (c) 2010-2012 OpenStack Foundation.
#
# Swift documentation build configuration file, created by
# sphinx-quickstart on Tue May 18 13:50:15 2010.
#
# This file is execfile()d with the current directory set to its containing
# dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import datetime

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.extend([os.path.abspath('../swift'), os.path.abspath('..'),
                 os.path.abspath('../bin')])

# -- General configuration ----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx',
              'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath',
              'sphinx.ext.ifconfig']
todo_include_todos = True

# Add any paths that contain templates here, relative to this directory.
# Changing the path so that the Hudson build output contains GA code and the
# source docs do not contain the code so local, offline sphinx builds are
# "clean."
templates_path = []
if os.getenv('HUDSON_PUBLISH_DOCS'):
    templates_path = ['_ga', '_templates']
else:
    templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Swift'
copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
from swift import __version__
version = __version__.rsplit('.', 1)[0]
# The full version, including alpha/beta/rc tags.
release = __version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = True

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ['swift.']


# -- Options for HTML output -----------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme = 'default'
html_theme_path = ["."]
html_theme = '_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# " v documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1"
html_last_updated_fmt = os.popen(git_cmd).read()

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a  tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'swiftdoc'


# -- Options for LaTeX output -------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
    ('index', 'Swift.tex', u'Swift Documentation',
     u'Swift Team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('http://docs.python.org/', None),
                       'nova': ('http://nova.openstack.org', None),
                       'glance': ('http://glance.openstack.org', None)}
swift-1.13.1/doc/source/account.rst0000664000175400017540000000117212323703611020321 0ustar  jenkinsjenkins00000000000000.. _account:

*******
Account
*******

.. _account-auditor:

Account Auditor
===============

.. automodule:: swift.account.auditor
    :members:
    :undoc-members:
    :show-inheritance:

.. _account-backend:

Account Backend
===============

.. automodule:: swift.account.backend
    :members:
    :undoc-members:
    :show-inheritance:

.. _account-reaper:

Account Reaper
==============

.. automodule:: swift.account.reaper
    :members:
    :undoc-members:
    :show-inheritance:

.. _account-server:

Account Server
==============

.. automodule:: swift.account.server
    :members:
    :undoc-members:
    :show-inheritance:
swift-1.13.1/doc/source/_theme/0000775000175400017540000000000012323703665017404 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/source/_theme/theme.conf0000664000175400017540000000012212323703611021337 0ustar  jenkinsjenkins00000000000000[theme]
inherit = sphinxdoc
stylesheet = sphinxdoc.css
pygments_style = friendly

swift-1.13.1/doc/source/_theme/layout.html0000664000175400017540000000523312323703611021601 0ustar  jenkinsjenkins00000000000000{% extends "sphinxdoc/layout.html" %}
{% set css_files = css_files + ['_static/tweaks.css'] %}

{%- macro sidebar() %}
{%- if not embedded %}{% if not theme_nosidebar|tobool %}
      

        

          {%- block sidebarlogo %}
          {%- if logo %}
            
          {%- endif %}
          {%- endblock %}
          {%- block sidebartoc %}
          {%- if display_toc %}
            
{{ _('Table Of Contents') }}

            {{ toc }}
          {%- endif %}
          {%- endblock %}
          {%- block sidebarrel %}
          {%- if prev %}
            
{{ _('Previous topic') }}

            
{{ prev.title }}

          {%- endif %}
          {%- if next %}
            
{{ _('Next topic') }}

            
{{ next.title }}

          {%- endif %}
          {%- endblock %}
          {%- block sidebarsourcelink %}
          {%- if show_source and has_source and sourcename %}
            
{{ _('This Page') }}

            
          {%- endif %}
          {%- endblock %}
          {%- if customsidebar %}
          {% include customsidebar %}
          {%- endif %}
          {%- block sidebarsearch %}
          {%- if pagename != "search" %}
          
          
          {%- endif %}

          {%- endblock %}
        

      

      {%- endif %}{% endif %}
{%- endmacro %}
swift-1.13.1/doc/source/overview_ring.rst0000664000175400017540000003513212323703611021555 0ustar  jenkinsjenkins00000000000000=========
The Rings
=========

The rings determine where data should reside in the cluster. There is a
separate ring for account databases, container databases, and individual
objects but each ring works in the same way. These rings are externally
managed, in that the server processes themselves do not modify the rings, they
are instead given new rings modified by other tools.

The ring uses a configurable number of bits from a path's MD5 hash as a
partition index that designates a device. The number of bits kept from the hash
is known as the partition power, and 2 to the partition power indicates the
partition count. Partitioning the full MD5 hash ring allows other parts of the
cluster to work in batches of items at once which ends up either more efficient
or at least less complex than working with each item separately or the entire
cluster all at once.

Another configurable value is the replica count, which indicates how many of
the partition->device assignments comprise a single ring. For a given partition
number, each replica's device will not be in the same zone as any other
replica's device. Zones can be used to group devices based on physical
locations, power separations, network separations, or any other attribute that
would lessen multiple replicas being unavailable at the same time.

------------
Ring Builder
------------

The rings are built and managed manually by a utility called the ring-builder.
The ring-builder assigns partitions to devices and writes an optimized Python
structure to a gzipped, serialized file on disk for shipping out to the servers.
The server processes just check the modification time of the file occasionally
and reload their in-memory copies of the ring structure as needed. Because of
how the ring-builder manages changes to the ring, using a slightly older ring
usually just means one of the three replicas for a subset of the partitions
will be incorrect, which can be easily worked around.

The ring-builder also keeps its own builder file with the ring information and
additional data required to build future rings. It is very important to keep
multiple backup copies of these builder files. One option is to copy the
builder files out to every server while copying the ring files themselves.
Another is to upload the builder files into the cluster itself. Complete loss
of a builder file will mean creating a new ring from scratch, nearly all
partitions will end up assigned to different devices, and therefore nearly all
data stored will have to be replicated to new locations. So, recovery from a
builder file loss is possible, but data will definitely be unreachable for an
extended time.

-------------------
Ring Data Structure
-------------------

The ring data structure consists of three top level fields: a list of devices
in the cluster, a list of lists of device ids indicating partition to device
assignments, and an integer indicating the number of bits to shift an MD5 hash
to calculate the partition for the hash.

***************
List of Devices
***************

The list of devices is known internally to the Ring class as devs. Each item in
the list of devices is a dictionary with the following keys:

======  =======  ==============================================================
id      integer  The index into the list devices.
zone    integer  The zone the devices resides in.
weight  float    The relative weight of the device in comparison to other
                 devices. This usually corresponds directly to the amount of
                 disk space the device has compared to other devices. For
                 instance a device with 1 terabyte of space might have a weight
                 of 100.0 and another device with 2 terabytes of space might
                 have a weight of 200.0. This weight can also be used to bring
                 back into balance a device that has ended up with more or less
                 data than desired over time. A good average weight of 100.0
                 allows flexibility in lowering the weight later if necessary.
ip      string   The IP address of the server containing the device.
port    int      The TCP port the listening server process uses that serves
                 requests for the device.
device  string   The on disk name of the device on the server.
                 For example: sdb1
meta    string   A general-use field for storing additional information for the
                 device. This information isn't used directly by the server
                 processes, but can be useful in debugging. For example, the
                 date and time of installation and hardware manufacturer could
                 be stored here.
======  =======  ==============================================================

Note: The list of devices may contain holes, or indexes set to None, for
devices that have been removed from the cluster. Generally, device ids are not
reused. Also, some devices may be temporarily disabled by setting their weight
to 0.0. To obtain a list of active devices (for uptime polling, for example)
the Python code would look like: ``devices = [device for device in self.devs if
device and device['weight']]``

*************************
Partition Assignment List
*************************

This is a list of array('H') of devices ids. The outermost list contains an
array('H') for each replica. Each array('H') has a length equal to the
partition count for the ring. Each integer in the array('H') is an index into
the above list of devices. The partition list is known internally to the Ring
class as _replica2part2dev_id.

So, to create a list of device dictionaries assigned to a partition, the Python
code would look like: ``devices = [self.devs[part2dev_id[partition]] for
part2dev_id in self._replica2part2dev_id]``

That code is a little simplistic, as it does not account for the
removal of duplicate devices. If a ring has more replicas than
devices, then a partition will have more than one replica on one
device; that's simply the pigeonhole principle at work.

array('H') is used for memory conservation as there may be millions of
partitions.

*******************
Fractional Replicas
*******************

A ring is not restricted to having an integer number of replicas. In order to
support the gradual changing of replica counts, the ring is able to have a real
number of replicas.

When the number of replicas is not an integer, then the last element of
_replica2part2dev_id will have a length that is less than the partition count
for the ring. This means that some partitions will have more replicas than
others. For example, if a ring has 3.25 replicas, then 25% of its partitions
will have four replicas, while the remaining 75% will have just three.


*********************
Partition Shift Value
*********************

The partition shift value is known internally to the Ring class as _part_shift.
This value used to shift an MD5 hash to calculate the partition on which the
data for that hash should reside. Only the top four bytes of the hash is used
in this process. For example, to compute the partition for the path
/account/container/object the Python code might look like: ``partition =
unpack_from('>I', md5('/account/container/object').digest())[0] >>
self._part_shift``

For a ring generated with part_power P, the partition shift value is
32 - P.

-----------------
Building the Ring
-----------------

The initial building of the ring first calculates the number of partitions that
should ideally be assigned to each device based the device's weight. For
example, given a partition power of 20, the ring will have 1,048,576 partitions.
If there are 1,000 devices of equal weight they will each desire 1,048.576
partitions. The devices are then sorted by the number of partitions they desire
and kept in order throughout the initialization process.

Note: each device is also assigned a random tiebreaker value that is used when
two devices desire the same number of partitions. This tiebreaker is not stored
on disk anywhere, and so two different rings created with the same parameters
will have different partition assignments. For repeatable partition assignments,
``RingBuilder.rebalance()`` takes an optional seed value that will be used to
seed Python's pseudo-random number generator.

Then, the ring builder assigns each replica of each partition to the device that
desires the most partitions at that point while keeping it as far away as
possible from other replicas. The ring builder prefers to assign a replica to a
device in a regions that has no replicas already; should there be no such region
available, the ring builder will try to find a device in a different zone; if
not possible, it will look on a different server; failing that, it will just
look for a device that has no replicas; finally, if all other options are
exhausted, the ring builder will assign the replica to the device that has the
fewest replicas already assigned. Note that assignment of multiple replicas to
one device will only happen if the ring has fewer devices than it has replicas.

When building a new ring based on an old ring, the desired number of partitions
each device wants is recalculated. Next the partitions to be reassigned are
gathered up. Any removed devices have all their assigned partitions unassigned
and added to the gathered list. Any partition replicas that (due to the
addition of new devices) can be spread out for better durability are unassigned
and added to the gathered list. Any devices that have more partitions than they
now desire have random partitions unassigned from them and added to the
gathered list. Lastly, the gathered partitions are then reassigned to devices
using a similar method as in the initial assignment described above.

Whenever a partition has a replica reassigned, the time of the reassignment is
recorded. This is taken into account when gathering partitions to reassign so
that no partition is moved twice in a configurable amount of time. This
configurable amount of time is known internally to the RingBuilder class as
min_part_hours. This restriction is ignored for replicas of partitions on
devices that have been removed, as removing a device only happens on device
failure and there's no choice but to make a reassignment.

The above processes don't always perfectly rebalance a ring due to the random
nature of gathering partitions for reassignment. To help reach a more balanced
ring, the rebalance process is repeated until near perfect (less 1% off) or
when the balance doesn't improve by at least 1% (indicating we probably can't
get perfect balance due to wildly imbalanced zones or too many partitions
recently moved).

-------
History
-------

The ring code went through many iterations before arriving at what it is now
and while it has been stable for a while now, the algorithm may be tweaked or
perhaps even fundamentally changed if new ideas emerge. This section will try
to describe the previous ideas attempted and attempt to explain why they were
discarded.

A "live ring" option was considered where each server could maintain its own
copy of the ring and the servers would use a gossip protocol to communicate the
changes they made. This was discarded as too complex and error prone to code
correctly in the project time span available. One bug could easily gossip bad
data out to the entire cluster and be difficult to recover from. Having an
externally managed ring simplifies the process, allows full validation of data
before it's shipped out to the servers, and guarantees each server is using a
ring from the same timeline. It also means that the servers themselves aren't
spending a lot of resources maintaining rings.

A couple of "ring server" options were considered. One was where all ring
lookups would be done by calling a service on a separate server or set of
servers, but this was discarded due to the latency involved. Another was much
like the current process but where servers could submit change requests to the
ring server to have a new ring built and shipped back out to the servers. This
was discarded due to project time constraints and because ring changes are
currently infrequent enough that manual control was sufficient. However, lack
of quick automatic ring changes did mean that other parts of the system had to
be coded to handle devices being unavailable for a period of hours until
someone could manually update the ring.

The current ring process has each replica of a partition independently assigned
to a device. A version of the ring that used a third of the memory was tried,
where the first replica of a partition was directly assigned and the other two
were determined by "walking" the ring until finding additional devices in other
zones. This was discarded as control was lost as to how many replicas for a
given partition moved at once. Keeping each replica independent allows for
moving only one partition replica within a given time window (except due to
device failures). Using the additional memory was deemed a good tradeoff for
moving data around the cluster much less often.

Another ring design was tried where the partition to device assignments weren't
stored in a big list in memory but instead each device was assigned a set of
hashes, or anchors. The partition would be determined from the data item's hash
and the nearest device anchors would determine where the replicas should be
stored. However, to get reasonable distribution of data each device had to have
a lot of anchors and walking through those anchors to find replicas started to
add up. In the end, the memory savings wasn't that great and more processing
power was used, so the idea was discarded.

A completely non-partitioned ring was also tried but discarded as the
partitioning helps many other parts of the system, especially replication.
Replication can be attempted and retried in a partition batch with the other
replicas rather than each data item independently attempted and retried. Hashes
of directory structures can be calculated and compared with other replicas to
reduce directory walking and network traffic.

Partitioning and independently assigning partition replicas also allowed for
the best balanced cluster. The best of the other strategies tended to give
+-10% variance on device balance with devices of equal weight and +-15% with
devices of varying weights. The current strategy allows us to get +-3% and +-8%
respectively.

Various hashing algorithms were tried. SHA offers better security, but the ring
doesn't need to be cryptographically secure and SHA is slower. Murmur was much
faster, but MD5 was built-in and hash computation is a small percentage of the
overall request handling time. In all, once it was decided the servers wouldn't
be maintaining the rings themselves anyway and only doing hash lookups, MD5 was
chosen for its general availability, good distribution, and adequate speed.
swift-1.13.1/doc/source/overview_object_versioning.rst0000664000175400017540000000703212323703611024325 0ustar  jenkinsjenkins00000000000000=================
Object Versioning
=================

--------
Overview
--------

Object versioning in swift is implemented by setting a flag on the container
to tell swift to version all objects in the container. The flag is the
``X-Versions-Location`` header on the container, and its value is the
container where the versions are stored. It is recommended to use a different
``X-Versions-Location`` container for each container that is being versioned.

When data is ``PUT`` into a versioned container (a container with the
versioning flag turned on), the existing data in the file is redirected to a
new object and the data in the ``PUT`` request is saved as the data for the
versioned object. The new object name (for the previous version) is
``//``, where ``length``
is the 3-character zero-padded hexidecimal length of the ```` and
```` is the timestamp of when the previous version was created.

A ``GET`` to a versioned object will return the current version of the object
without having to do any request redirects or metadata lookups.

A ``POST`` to a versioned object will update the object metadata as normal,
but will not create a new version of the object. In other words, new versions
are only created when the content of the object changes.

A ``DELETE`` to a versioned object will only remove the current version of the
object. If you have 5 total versions of the object, you must delete the
object 5 times to completely remove the object.

Note: A large object manifest file cannot be versioned, but a large object
manifest may point to versioned segments.

--------------------------------------------------
How to Enable Object Versioning in a Swift Cluster
--------------------------------------------------

Set ``allow_versions`` to ``True`` in the container server config.

-----------------------
Examples Using ``curl``
-----------------------

First, create a container with the ``X-Versions-Location`` header or add the
header to an existing container. Also make sure the container referenced by
the ``X-Versions-Location`` exists. In this example, the name of that
container is "versions"::

    curl -i -XPUT -H "X-Auth-Token: " \
        -H "X-Versions-Location: versions" http:///container
    curl -i -XPUT -H "X-Auth-Token: " http:///versions

Create an object (the first version)::

    curl -i -XPUT --data-binary 1 -H "X-Auth-Token: " \
        http:///container/myobject

Now create a new version of that object::

    curl -i -XPUT --data-binary 2 -H "X-Auth-Token: " \
        http:///container/myobject

See a listing of the older versions of the object::

    curl -i -H "X-Auth-Token: " \
        http:///versions?prefix=008myobject/

Now delete the current version of the object and see that the older version is
gone::

    curl -i -XDELETE -H "X-Auth-Token: " \
        http:///container/myobject
    curl -i -H "X-Auth-Token: " \
        http:///versions?prefix=008myobject/

---------------------------------------------------
How to Disable Object Versioning in a Swift Cluster
---------------------------------------------------

If you want to disable all functionality, set ``allow_versions`` back to
``False`` in the container server config.

Disable versioning a versioned container (x is any value except empty)::

    curl -i -HPOST -H "X-Auth-Token: " \
        -H "X-Remove-Versions-Location: x" http:///container
swift-1.13.1/doc/source/_ga/0000775000175400017540000000000012323703665016671 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/source/_ga/layout.html0000664000175400017540000000102612323703611021062 0ustar  jenkinsjenkins00000000000000{% extends "!layout.html" %}

{% block footer %}
{{ super() }}


{% endblock %}

swift-1.13.1/doc/source/proxy.rst0000664000175400017540000000131112323703611020041 0ustar  jenkinsjenkins00000000000000.. _proxy:

*****
Proxy
*****

.. _proxy-controllers:

Proxy Controllers
=================

Base
~~~~

.. automodule:: swift.proxy.controllers.base
    :members:
    :undoc-members:
    :show-inheritance:

Account
~~~~~~~

.. automodule:: swift.proxy.controllers.account
    :members:
    :undoc-members:
    :show-inheritance:

Container
~~~~~~~~~

.. automodule:: swift.proxy.controllers.container
    :members:
    :undoc-members:
    :show-inheritance:

Object
~~~~~~

.. automodule:: swift.proxy.controllers.obj
    :members:
    :undoc-members:
    :show-inheritance:

.. _proxy-server:

Proxy Server
============

.. automodule:: swift.proxy.server
    :members:
    :undoc-members:
    :show-inheritance:
swift-1.13.1/doc/source/development_auth.rst0000664000175400017540000004643012323703611022236 0ustar  jenkinsjenkins00000000000000==========================
Auth Server and Middleware
==========================

--------------------------------------------
Creating Your Own Auth Server and Middleware
--------------------------------------------

The included swift/common/middleware/tempauth.py is a good example of how to
create an auth subsystem with proxy server auth middleware. The main points are
that the auth middleware can reject requests up front, before they ever get to
the Swift Proxy application, and afterwards when the proxy issues callbacks to
verify authorization.

It's generally good to separate the authentication and authorization
procedures. Authentication verifies that a request actually comes from who it
says it does. Authorization verifies the 'who' has access to the resource(s)
the request wants.

Authentication is performed on the request before it ever gets to the Swift
Proxy application. The identity information is gleaned from the request,
validated in some way, and the validation information is added to the WSGI
environment as needed by the future authorization procedure. What exactly is
added to the WSGI environment is solely dependent on what the installed
authorization procedures need; the Swift Proxy application itself needs no
specific information, it just passes it along. Convention has
environ['REMOTE_USER'] set to the authenticated user string but often more
information is needed than just that.

The included TempAuth will set the REMOTE_USER to a comma separated list of
groups the user belongs to. The first group will be the "user's group", a group
that only the user belongs to. The second group will be the "account's group",
a group that includes all users for that auth account (different than the
storage account). The third group is optional and is the storage account
string. If the user does not have admin access to the account, the third group
will be omitted.

It is highly recommended that authentication server implementers prefix their
tokens and Swift storage accounts they create with a configurable reseller
prefix (`AUTH_` by default with the included TempAuth). This prefix will avoid
conflicts with other authentication servers that might be using the same
Swift cluster. Otherwise, the Swift cluster will have to try all the resellers
until one validates a token or all fail.

A restriction with group names is that no group name should begin with a period
'.' as that is reserved for internal Swift use (such as the .r for referrer
designations as you'll see later).

Example Authentication with TempAuth:

    * Token AUTH_tkabcd is given to the TempAuth middleware in a request's
      X-Auth-Token header.
    * The TempAuth middleware validates the token AUTH_tkabcd and discovers
      it matches the "tester" user within the "test" account for the storage
      account "AUTH_storage_xyz".
    * The TempAuth middleware sets the REMOTE_USER to
      "test:tester,test,AUTH_storage_xyz"
    * Now this user will have full access (via authorization procedures later)
      to the AUTH_storage_xyz Swift storage account and access to containers in
      other storage accounts, provided the storage account begins with the same
      `AUTH_` reseller prefix and the container has an ACL specifying at least
      one of those three groups.

Authorization is performed through callbacks by the Swift Proxy server to the
WSGI environment's swift.authorize value, if one is set. The swift.authorize
value should simply be a function that takes a Request as an argument and
returns None if access is granted or returns a callable(environ,
start_response) if access is denied. This callable is a standard WSGI callable.
Generally, you should return 403 Forbidden for requests by an authenticated
user and 401 Unauthorized for an unauthenticated request. For example, here's
an authorize function that only allows GETs (in this case you'd probably return
405 Method Not Allowed, but ignore that for the moment).::

    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    def authorize(req):
        if req.method == 'GET':
            return None
        if req.remote_user:
            return HTTPForbidden(request=req)
        else:
            return HTTPUnauthorized(request=req)

Adding the swift.authorize callback is often done by the authentication
middleware as authentication and authorization are often paired together. But,
you could create separate authorization middleware that simply sets the
callback before passing on the request. To continue our example above::

    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    class Authorization(object):

        def __init__(self, app, conf):
            self.app = app
            self.conf = conf

        def __call__(self, environ, start_response):
            environ['swift.authorize'] = self.authorize
            return self.app(environ, start_response)

        def authorize(self, req):
            if req.method == 'GET':
                return None
            if req.remote_user:
                return HTTPForbidden(request=req)
            else:
                return HTTPUnauthorized(request=req)


    def filter_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def auth_filter(app):
            return Authorization(app, conf)
        return auth_filter

The Swift Proxy server will call swift.authorize after some initial work, but
before truly trying to process the request. Positive authorization at this
point will cause the request to be fully processed immediately. A denial at
this point will immediately send the denial response for most operations.

But for some operations that might be approved with more information, the
additional information will be gathered and added to the WSGI environment and
then swift.authorize will be called once more. These are called delay_denial
requests and currently include container read requests and object read and
write requests. For these requests, the read or write access control string
(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl'
attribute in the Request passed to swift.authorize.

The delay_denial procedures allow skipping possibly expensive access control
string retrievals for requests that can be approved without that information,
such as administrator or account owner requests.

To further our example, we now will approve all requests that have the access
control string set to same value as the authenticated user string. Note that
you probably wouldn't do this exactly as the access control string represents a
list rather than a single user, but it'll suffice for this example::

    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    class Authorization(object):

        def __init__(self, app, conf):
            self.app = app
            self.conf = conf

        def __call__(self, environ, start_response):
            environ['swift.authorize'] = self.authorize
            return self.app(environ, start_response)

        def authorize(self, req):
            # Allow anyone to perform GET requests
            if req.method == 'GET':
                return None
            # Allow any request where the acl equals the authenticated user
            if getattr(req, 'acl', None) == req.remote_user:
                return None
            if req.remote_user:
                return HTTPForbidden(request=req)
            else:
                return HTTPUnauthorized(request=req)


    def filter_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def auth_filter(app):
            return Authorization(app, conf)
        return auth_filter

The access control string has a standard format included with Swift, though
this can be overridden if desired. The standard format can be parsed with
swift.common.middleware.acl.parse_acl which converts the string into two arrays
of strings: (referrers, groups). The referrers allow comparing the request's
Referer header to control access. The groups allow comparing the
request.remote_user (or other sources of group information) to control access.
Checking referrer access can be accomplished by using the
swift.common.middleware.acl.referrer_allowed function. Checking group access is
usually a simple string comparison.

Let's continue our example to use parse_acl and referrer_allowed. Now we'll
only allow GETs after a referrer check and any requests after a group check::

    from swift.common.middleware.acl import parse_acl, referrer_allowed
    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    class Authorization(object):

        def __init__(self, app, conf):
            self.app = app
            self.conf = conf

        def __call__(self, environ, start_response):
            environ['swift.authorize'] = self.authorize
            return self.app(environ, start_response)

        def authorize(self, req):
            if hasattr(req, 'acl'):
                referrers, groups = parse_acl(req.acl)
                if req.method == 'GET' and referrer_allowed(req, referrers):
                    return None
                if req.remote_user and groups and req.remote_user in groups:
                    return None
            if req.remote_user:
                return HTTPForbidden(request=req)
            else:
                return HTTPUnauthorized(request=req)


    def filter_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def auth_filter(app):
            return Authorization(app, conf)
        return auth_filter

The access control strings are set with PUTs and POSTs to containers
with the X-Container-Read and X-Container-Write headers. Swift allows
these strings to be set to any value, though it's very useful to
validate that the strings meet the desired format and return a useful
error to the user if they don't.

To support this validation, the Swift Proxy application will call the WSGI
environment's swift.clean_acl callback whenever one of these headers is to be
written. The callback should take a header name and value as its arguments. It
should return the cleaned value to save if valid or raise a ValueError with a
reasonable error message if not.

There is an included swift.common.middleware.acl.clean_acl that validates the
standard Swift format. Let's improve our example by making use of that::

    from swift.common.middleware.acl import \
        clean_acl, parse_acl, referrer_allowed
    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    class Authorization(object):

        def __init__(self, app, conf):
            self.app = app
            self.conf = conf

        def __call__(self, environ, start_response):
            environ['swift.authorize'] = self.authorize
            environ['swift.clean_acl'] = clean_acl
            return self.app(environ, start_response)

        def authorize(self, req):
            if hasattr(req, 'acl'):
                referrers, groups = parse_acl(req.acl)
                if req.method == 'GET' and referrer_allowed(req, referrers):
                    return None
                if req.remote_user and groups and req.remote_user in groups:
                    return None
            if req.remote_user:
                return HTTPForbidden(request=req)
            else:
                return HTTPUnauthorized(request=req)


    def filter_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def auth_filter(app):
            return Authorization(app, conf)
        return auth_filter

Now, if you want to override the format for access control strings you'll have
to provide your own clean_acl function and you'll have to do your own parsing
and authorization checking for that format. It's highly recommended you use the
standard format simply to support the widest range of external tools, but
sometimes that's less important than meeting certain ACL requirements.


----------------------------
Integrating With repoze.what
----------------------------

Here's an example of integration with repoze.what, though honestly I'm no
repoze.what expert by any stretch; this is just included here to hopefully give
folks a start on their own code if they want to use repoze.what::

    from time import time

    from eventlet.timeout import Timeout
    from repoze.what.adapters import BaseSourceAdapter
    from repoze.what.middleware import setup_auth
    from repoze.what.predicates import in_any_group, NotAuthorizedError
    from swift.common.bufferedhttp import http_connect_raw as http_connect
    from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
    from swift.common.utils import cache_from_env, split_path
    from swift.common.swob import HTTPForbidden, HTTPUnauthorized


    class DevAuthorization(object):

        def __init__(self, app, conf):
            self.app = app
            self.conf = conf

        def __call__(self, environ, start_response):
            environ['swift.authorize'] = self.authorize
            environ['swift.clean_acl'] = clean_acl
            return self.app(environ, start_response)

        def authorize(self, req):
            version, account, container, obj = split_path(req.path, 1, 4, True)
            if not account:
                return self.denied_response(req)
            referrers, groups = parse_acl(getattr(req, 'acl', None))
            if referrer_allowed(req, referrers):
                return None
            try:
                in_any_group(account, *groups).check_authorization(req.environ)
            except NotAuthorizedError:
                return self.denied_response(req)
            return None

        def denied_response(self, req):
            if req.remote_user:
                return HTTPForbidden(request=req)
            else:
                return HTTPUnauthorized(request=req)


    class DevIdentifier(object):

        def __init__(self, conf):
            self.conf = conf

        def identify(self, env):
            return {'token':
                    env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))}

        def remember(self, env, identity):
            return []

        def forget(self, env, identity):
            return []


    class DevAuthenticator(object):

        def __init__(self, conf):
            self.conf = conf
            self.auth_host = conf.get('ip', '127.0.0.1')
            self.auth_port = int(conf.get('port', 11000))
            self.ssl = \
                conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes')
            self.auth_prefix = conf.get('prefix', '/')
            self.timeout = int(conf.get('node_timeout', 10))

        def authenticate(self, env, identity):
            token = identity.get('token')
            if not token:
                return None
            memcache_client = cache_from_env(env)
            key = 'devauth/%s' % token
            cached_auth_data = memcache_client.get(key)
            if cached_auth_data:
                start, expiration, user = cached_auth_data
                if time() - start <= expiration:
                    return user
            with Timeout(self.timeout):
                conn = http_connect(self.auth_host, self.auth_port, 'GET',
                        '%stoken/%s' % (self.auth_prefix, token), ssl=self.ssl)
                resp = conn.getresponse()
                resp.read()
                conn.close()
            if resp.status == 204:
                expiration = float(resp.getheader('x-auth-ttl'))
                user = resp.getheader('x-auth-user')
                memcache_client.set(key, (time(), expiration, user),
                                    timeout=expiration)
                return user
            return None


    class DevChallenger(object):

        def __init__(self, conf):
            self.conf = conf

        def challenge(self, env, status, app_headers, forget_headers):
            def no_challenge(env, start_response):
                start_response(str(status), [])
                return []
            return no_challenge


    class DevGroupSourceAdapter(BaseSourceAdapter):

        def __init__(self, *args, **kwargs):
            super(DevGroupSourceAdapter, self).__init__(*args, **kwargs)
            self.sections = {}

        def _get_all_sections(self):
            return self.sections

        def _get_section_items(self, section):
            return self.sections[section]

        def _find_sections(self, credentials):
            return credentials['repoze.what.userid'].split(',')

        def _include_items(self, section, items):
            self.sections[section] |= items

        def _exclude_items(self, section, items):
            for item in items:
                self.sections[section].remove(item)

        def _item_is_included(self, section, item):
            return item in self.sections[section]

        def _create_section(self, section):
            self.sections[section] = set()

        def _edit_section(self, section, new_section):
            self.sections[new_section] = self.sections[section]
            del self.sections[section]

        def _delete_section(self, section):
            del self.sections[section]

        def _section_exists(self, section):
            return self.sections.has_key(section)


    class DevPermissionSourceAdapter(BaseSourceAdapter):

        def __init__(self, *args, **kwargs):
            super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs)
            self.sections = {}

        def _get_all_sections(self):
            return self.sections

        def _get_section_items(self, section):
            return self.sections[section]

        def _find_sections(self, group_name):
            return set([n for (n, p) in self.sections.items()
                        if group_name in p])

        def _include_items(self, section, items):
            self.sections[section] |= items

        def _exclude_items(self, section, items):
            for item in items:
                self.sections[section].remove(item)

        def _item_is_included(self, section, item):
            return item in self.sections[section]

        def _create_section(self, section):
            self.sections[section] = set()

        def _edit_section(self, section, new_section):
            self.sections[new_section] = self.sections[section]
            del self.sections[section]

        def _delete_section(self, section):
            del self.sections[section]

        def _section_exists(self, section):
            return self.sections.has_key(section)


    def filter_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def auth_filter(app):
            return setup_auth(DevAuthorization(app, conf),
                group_adapters={'all_groups': DevGroupSourceAdapter()},
                permission_adapters={'all_perms': DevPermissionSourceAdapter()},
                identifiers=[('devauth', DevIdentifier(conf))],
                authenticators=[('devauth', DevAuthenticator(conf))],
                challengers=[('devauth', DevChallenger(conf))])
        return auth_filter

-----------------------
Allowing CORS with Auth
-----------------------

Cross Origin RequestS require that the auth system allow the OPTIONS method to 
pass through without a token.  The preflight request will make an OPTIONS call 
against the object or container and will not work if the auth system stops it.
See TempAuth for an example of how OPTIONS requests are handled.
swift-1.13.1/doc/source/associated_projects.rst0000664000175400017540000000667312323703611022730 0ustar  jenkinsjenkins00000000000000.. _associated_projects:

Associated Projects
===================


Application Bindings
--------------------

* OpenStack supported binding:

   * `Python-SwiftClient `_

* Unofficial libraries and bindings:

    * `PHP-opencloud `_ - Official Rackspace PHP bindings that should work for other Swift deployments too.
    * `PyRAX `_ - Official Rackspace Python bindings for CloudFiles that should work for other Swift deployments too.
    * `openstack.net `_ - Official Rackspace .NET bindings that should work for other Swift deployments too.
    * `RSwift `_ - R API bindings.
    * `Go language bindings `_
    * `supload `_ - Bash script to upload file to cloud storage based on OpenStack Swift API.
    * `libcloud `_ - Apache Libcloud - a unified interface in Python for different clouds with OpenStack Swift support.
    * `SwiftBox `_ - C# library using RestSharp
    * `jclouds `_ - Java library offering bindings for all OpenStack projects
    * `java-openstack-swift `_ - Java bindings for OpenStack Swift

Authentication
--------------

* `Keystone `_ - Official Identity Service for OpenStack.
* `Swauth `_ - Older Swift authentication service that only requires Swift itself.
* `Basicauth `_ - HTTP Basic authentication support (keystone backed).


Command Line Access
-------------------

* `Swiftly `_ - Alternate command line access to Swift with direct (no proxy) access capabilities as well.


Log Processing
--------------

* `Slogging `_ - Basic stats and logging tools.


Monitoring & Statistics
-----------------------

* `Swift Informant `_ - Swift Proxy Middleware to send events to a statsd instance.


Content Distribution Network Integration
----------------------------------------

* `SOS `_ - Swift Origin Server.


Alternative API
---------------

* `Swift3 `_ - Amazon S3 API emulation.
* `CDMI `_ - CDMI support


.. _custom-logger-hooks-label:

Custom Logger Hooks
-------------------

* `swift-sentry `_ - Sentry exception reporting for Swift

Other
-----

* `Glance `_ - Provides services for discovering, registering, and retrieving virtual machine images (for OpenStack Compute [Nova], for example).
* `Better Staticweb `_ - Makes swift containers accessible by default.
* `Swiftsync `_ - A massive syncer between two swift clusters.
* `Swiftbrowser `_ - Simple web app to access Openstack Swift.
* `Swift-account-stats `_ - Swift-account-stats is a tool to report statistics on Swift usage at tenant and global levels.
swift-1.13.1/doc/source/misc.rst0000664000175400017540000000316012323703611017617 0ustar  jenkinsjenkins00000000000000.. _misc:

****
Misc
****

.. _acls:

ACLs
====

.. automodule:: swift.common.middleware.acl
    :members:
    :show-inheritance:

.. _buffered_http:

Buffered HTTP
=============

.. automodule:: swift.common.bufferedhttp
    :members:
    :show-inheritance:

.. _constraints:

Constraints
===========

.. automodule:: swift.common.constraints
    :members:
    :undoc-members:
    :show-inheritance:

Container Sync Realms
=====================

.. automodule:: swift.common.container_sync_realms
    :members:
    :show-inheritance:

.. _direct_client:

Direct Client
=============

.. automodule:: swift.common.direct_client
    :members:
    :undoc-members:
    :show-inheritance:

.. _exceptions:

Exceptions
==========

.. automodule:: swift.common.exceptions
    :members:
    :undoc-members:
    :show-inheritance:

.. _internal_client:

Internal Client
===============

.. automodule:: swift.common.internal_client
    :members:
    :undoc-members:
    :show-inheritance:

Manager
=========

.. automodule:: swift.common.manager
    :members:
    :show-inheritance:

MemCacheD
=========

.. automodule:: swift.common.memcached
    :members:
    :show-inheritance:

.. _request_helpers:

Request Helpers
===============

.. automodule:: swift.common.request_helpers
    :members:
    :undoc-members:
    :show-inheritance:

.. _swob:

Swob
====

.. automodule:: swift.common.swob
    :members:
    :show-inheritance:
    :special-members: __call__

.. _utils:

Utils
=====

.. automodule:: swift.common.utils
    :members:
    :show-inheritance:

.. _wsgi:

WSGI
====

.. automodule:: swift.common.wsgi
    :members:
    :show-inheritance:
swift-1.13.1/doc/source/index.rst0000664000175400017540000000466612323703611020007 0ustar  jenkinsjenkins00000000000000..
      Copyright 2010-2012 OpenStack Foundation
      All Rights Reserved.

      Licensed under the Apache License, Version 2.0 (the "License"); you may
      not use this file except in compliance with the License. You may obtain
      a copy of the License at

          http://www.apache.org/licenses/LICENSE-2.0

      Unless required by applicable law or agreed to in writing, software
      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
      License for the specific language governing permissions and limitations
      under the License.

Welcome to Swift's documentation!
=================================

Swift is a highly available, distributed, eventually consistent object/blob
store. Organizations can use Swift to store lots of data efficiently, safely, and cheaply.

This documentation is generated by the Sphinx toolkit and lives in the source
tree.  Additional documentation on Swift and other components of OpenStack can
be found on the `OpenStack wiki`_ and at http://docs.openstack.org.

.. _`OpenStack wiki`: http://wiki.openstack.org

.. note::

    If you're looking for associated projects that enhance or use Swift, please see the :ref:`associated_projects` page.


.. toctree::
    :maxdepth: 1

    getting_started

Overview and Concepts
=====================

.. toctree::
    :maxdepth: 1

    Swift's API docs 
    overview_architecture
    overview_ring
    overview_reaper
    overview_auth
    overview_replication
    ratelimit
    overview_large_objects
    overview_object_versioning
    overview_container_sync
    overview_expiring_objects
    cors
    crossdomain
    associated_projects

Developer Documentation
=======================

.. toctree::
    :maxdepth: 1

    development_guidelines
    development_saio
    development_auth
    development_middleware
    development_ondisk_backends

Administrator Documentation
===========================

.. toctree::
    :maxdepth: 1

    howto_installmultinode
    deployment_guide
    apache_deployment_guide
    admin_guide
    replication_network
    logs

Source Documentation
====================

.. toctree::
    :maxdepth: 2

    ring
    proxy
    account
    container
    db
    object
    misc
    middleware


Indices and tables
==================

* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
swift-1.13.1/doc/source/howto_installmultinode.rst0000664000175400017540000004232312323703611023477 0ustar  jenkinsjenkins00000000000000==============================================================
Instructions for a Multiple Server Swift Installation (Ubuntu)
==============================================================

Prerequisites
-------------
* Ubuntu Server 10.04 LTS installation media

.. note:
    Swift can run with other distros, but for this document we will focus
    on installing on Ubuntu Server, ypmv (your packaging may vary).

Basic architecture and terms
----------------------------
- *node* - a host machine running one or more Swift services
- *Proxy node* - node that runs Proxy services; also runs TempAuth
- *Storage node* - node that runs Account, Container, and Object services
- *ring* - a set of mappings of Swift data to physical devices

This document shows a cluster using the following types of nodes:

- one Proxy node

  - Runs the swift-proxy-server processes which proxy requests to the
    appropriate Storage nodes. The proxy server will also contain
    the TempAuth service as WSGI middleware.

- five Storage nodes

  - Runs the swift-account-server, swift-container-server, and
    swift-object-server processes which control storage of the account
    databases, the container databases, as well as the actual stored
    objects.

.. note::
    Fewer Storage nodes can be used initially, but a minimum of 5 is
    recommended for a production cluster.

This document describes each Storage node as a separate zone in the ring.
It is recommended to have a minimum of 5 zones. A zone is a group of nodes
that is as isolated as possible from other nodes (separate servers, network,
power, even geography). The ring guarantees that every replica is stored
in a separate zone.  For more information about the ring and zones, see: :doc:`The Rings `.

To increase reliability, you may want to add additional Proxy servers for performance which is described in :ref:`add-proxy-server`.

Network Setup Notes
-------------------

This document refers to two networks.  An external network for connecting to the Proxy server, and a storage network that is not accessibile from outside the cluster, to which all of the nodes are connected.  All of the Swift services, as well as the rsync daemon on the Storage nodes are configured to listen on their STORAGE_LOCAL_NET IP addresses.

.. note::
    Run all commands as the root user

General OS configuration and partitioning for each node
-------------------------------------------------------

#. Install the baseline Ubuntu Server 10.04 LTS on all nodes.

#. Install common Swift software prereqs::

        apt-get install python-software-properties
        add-apt-repository ppa:swift-core/release
        apt-get update
        apt-get install swift python-swiftclient openssh-server

#. Create and populate configuration directories::

        mkdir -p /etc/swift
        chown -R swift:swift /etc/swift/

#. On the first node only, create /etc/swift/swift.conf::

        cat >/etc/swift/swift.conf </etc/swift/proxy-server.conf <> /etc/fstab
        mkdir -p /srv/node/sdb1
        mount /srv/node/sdb1
        chown swift:swift /srv/node/sdb1

#. Create /etc/rsyncd.conf::

        cat >/etc/rsyncd.conf </etc/swift/account-server.conf </etc/swift/container-server.conf </etc/swift/object-server.conf <' 

#. Check that ``swift`` works  (at this point, expect zero containers, zero objects, and zero bytes)::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat

#. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles'::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz
        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz

#. Use ``swift`` to download all files from the 'myfiles' container::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles

#. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder

#. Use ``swift`` to list your containers::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list

#. Use ``swift`` to list the contents of your 'builders' container::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders

#. Use ``swift`` to download all files from the 'builders' container::

        swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders

.. _add-proxy-server:

Adding a Proxy Server
---------------------

For reliability's sake you may want to have more than one proxy server. You can set up the additional proxy node in the same manner that you set up the first proxy node but with additional configuration steps.

Once you have more than two proxies, you also want to load balance between the two, which means your storage endpoint also changes. You can select from different strategies for load balancing. For example, you could use round robin dns, or an actual load balancer (like pound) in front of the two proxies, and point your storage url to the load balancer.

See :ref:`config-proxy` for the initial setup, and then follow these additional steps.

#. Update the list of memcache servers in /etc/swift/proxy-server.conf for all the added proxy servers. If you run multiple memcache servers, use this pattern for the multiple IP:port listings: `10.1.2.3:11211,10.1.2.4:11211` in each proxy server's conf file.::

        [filter:cache]
        use = egg:swift#memcache
        memcache_servers = $PROXY_LOCAL_NET_IP:11211

#. Change the storage url for any users to point to the load balanced url, rather than the first proxy server you created in /etc/swift/proxy-server.conf::

        [filter:tempauth]
        use = egg:swift#tempauth
        user_system_root = testpass .admin http[s]://:/v1/AUTH_system

#. Next, copy all the ring information to all the nodes, including your new proxy nodes, and ensure the ring info gets to all the storage nodes as well.

#. After you sync all the nodes, make sure the admin has the keys in /etc/swift and the ownership for the ring file is correct.

Troubleshooting Notes
---------------------
If you see problems, look in var/log/syslog (or messages on some distros).

Also, at Rackspace we have seen hints at drive failures by looking at error messages in /var/log/kern.log.

There are more debugging hints and tips in the :doc:`admin_guide`.
swift-1.13.1/doc/source/cors.rst0000664000175400017540000000765412323703611017646 0ustar  jenkinsjenkins00000000000000====
CORS
====

CORS_ is a mechanisim to allow code running in a browser (Javascript for
example) make requests to a domain other then the one from where it originated.

Swift supports CORS requests to containers and objects.

CORS metadata is held on the container only. The values given apply to the
container itself and all objects within it.

The supported headers are,

+------------------------------------------------+------------------------------+
| Metadata                                       | Use                          |
+================================================+==============================+
| X-Container-Meta-Access-Control-Allow-Origin   | Origins to be allowed to     |
|                                                | make Cross Origin Requests,  |
|                                                | space separated.             |
+------------------------------------------------+------------------------------+
| X-Container-Meta-Access-Control-Max-Age        | Max age for the Origin to    |
|                                                | hold the preflight results.  |
+------------------------------------------------+------------------------------+
| X-Container-Meta-Access-Control-Expose-Headers | Headers exposed to the user  |
|                                                | agent (e.g. browser) in the  |
|                                                | the actual request response. |
|                                                | Space separated.             |
+------------------------------------------------+------------------------------+

Before a browser issues an actual request it may issue a `preflight request`_.
The preflight request is an OPTIONS call to verify the Origin is allowed to
make the request. The sequence of events are,

* Browser makes OPTIONS request to Swift
* Swift returns 200/401 to browser based on allowed origins
* If 200, browser makes the "actual request" to Swift, i.e. PUT, POST, DELETE,
  HEAD, GET

When a browser receives a response to an actual request it only exposes those
headers listed in the ``Access-Control-Expose-Headers`` header. By default Swift
returns the following values for this header,

* "simple response headers" as listed on
  http://www.w3.org/TR/cors/#simple-response-header
* the headers ``etag``, ``x-timestamp``, ``x-trans-id``
* all metadata headers (``X-Container-Meta-*`` for containers and
  ``X-Object-Meta-*`` for objects)
* headers listed in ``X-Container-Meta-Access-Control-Expose-Headers``


-----------------
Sample Javascript
-----------------

To see some CORS Javascript in action download the `test CORS page`_ (source
below). Host it on a webserver and take note of the protocol and hostname
(origin) you'll be using to request the page, e.g. http://localhost.

Locate a container you'd like to query. Needless to say the Swift cluster
hosting this container should have CORS support. Append the origin of the
test page to the container's ``X-Container-Meta-Access-Control-Allow-Origin``
header,::

    curl -X POST -H 'X-Auth-Token: xxx' \
      -H 'X-Container-Meta-Access-Control-Allow-Origin: http://localhost' \
      http://192.168.56.3:8080/v1/AUTH_test/cont1

At this point the container is now accessible to CORS clients hosted on
http://localhost. Open the test CORS page in your browser.

#. Populate the Token field
#. Populate the URL field with the URL of either a container or object
#. Select the request method
#. Hit Submit

Assuming the request succeeds you should see the response header and body. If
something went wrong the response status will be 0.

.. _test CORS page:

--------------
Test CORS Page
--------------

A sample cross-site test page is located in the project source tree
``doc/source/test-cors.html``.

     .. literalinclude:: test-cors.html

.. _CORS: https://developer.mozilla.org/en-US/docs/HTTP/Access_control_CORS
.. _preflight request: https://developer.mozilla.org/en-US/docs/HTTP/Access_control_CORS#Preflighted_requests
swift-1.13.1/doc/source/logs.rst0000664000175400017540000001356112323703614017641 0ustar  jenkinsjenkins00000000000000====
Logs
====

Swift has quite verbose logging, and the generated logs can be used for
cluster monitoring, utilization calculations, audit records, and more. As an
overview, Swift's logs are sent to syslog and organized by log level and
syslog facility. All log lines related to the same request have the same
transaction id. This page documents the log formats used in the system.

----------
Proxy Logs
----------

The proxy logs contain the record of all external API requests made to the
proxy server. Swift's proxy servers log requests using a custom format
designed to provide robust information and simple processing. The log format
is::

    client_ip remote_addr datetime request_method request_path protocol
        status_int referer user_agent auth_token bytes_recvd bytes_sent
        client_etag transaction_id headers request_time source log_info
        request_start_time request_end_time

=================== ==========================================================
**Log Field**       **Value**
------------------- ----------------------------------------------------------
client_ip           Swift's guess at the end-client IP, taken from various
                    headers in the request.
remote_addr         The IP address of the other end of the TCP connection.
datetime            Timestamp of the request, in
                    day/month/year/hour/minute/second format.
request_method      The HTTP verb in the request.
request_path        The path portion of the request.
protocol            The transport protocol used (currently one of http or
                    https).
status_int          The response code for the request.
referer             The value of the HTTP Referer header.
user_agent          The value of the HTTP User-Agent header.
auth_token          The value of the auth token. This may be truncated or
                    otherwise obscured.
bytes_recvd         The number of bytes read from the client for this request.
bytes_sent          The number of bytes sent to the client in the body of the
                    response. This is how many bytes were yielded to the WSGI
                    server.
client_etag         The etag header value given by the client.
transaction_id      The transaction id of the request.
headers             The headers given in the request.
request_time        The duration of the request.
source              The "source" of the reuqest. This may be set for requests
                    that are generated in order to fulfill client requests,
                    e.g. bulk uploads.
log_info            Various info that may be useful for diagnostics, e.g. the
                    value of any x-delete-at header.
request_start_time  High-resolution timestamp from the start of the request.
request_end_time    High-resolution timestamp from the end of the request.
=================== ==========================================================

In one log line, all of the above fields are space-separated and url-encoded.
If any value is empty, it will be logged as a "-". This allows for simple
parsing by splitting each line on whitespace. New values may be placed at the
end of the log line from time to time, but the order of the existing values
will not change. Swift log processing utilities should look for the first N
fields they require (e.g. in Python using something like
``log_line.split()[:14]`` to get up through the transaction id).

Swift Source
============

The ``source`` value in the proxy logs is used to identify the originator of a
request in the system. For example, if the client initiates a bulk upload, the
proxy server may end up doing many requests. The initial bulk upload request
will be logged as normal, but all of the internal "child requests" will have a
source value indicating they came from the bulk functionality.

======================= =============================
**Logged Source Value** **Originator of the Request**
----------------------- -----------------------------
FP                      :ref:`formpost`
SLO                     :ref:`static-large-objects`
SW                      :ref:`staticweb`
TU                      :ref:`tempurl`
BD                      :ref:`bulk` (delete)
EA                      :ref:`bulk` (extract)
CQ                      :ref:`container-quotas`
CS                      :ref:`container-sync`
TA                      :ref:`common_tempauth`
DLO                     :ref:`dynamic-large-objects`
======================= =============================


-----------------
Storage Node Logs
-----------------

Swift's account, container, and object server processes each log requests
that they receive, if they have been configured to do so with the
``log_requests`` config parameter (which defaults to true). The format for
these log lines is::

    remote_addr - - [datetime] "request_method request_path" status_int
        content_length "referer" "transaction_id" "user_agent" request_time

=================== ==========================================================
**Log Field**       **Value**
------------------- ----------------------------------------------------------
remote_addr         The IP address of the other end of the TCP connection.
datetime            Timestamp of the request, in
                    "day/month/year:hour:minute:second +0000" format.
request_method      The HTTP verb in the request.
request_path        The path portion of the request.
status_int          The response code for the request.
content_length      The value of the Content-Length header in the response.
referer             The value of the HTTP Referer header.
transaction_id      The transaction id of the request.
user_agent          The value of the HTTP User-Agent header. Swift's proxy
                    server sets its user-agent to
                    ``"proxy-server ".``
request_time        The duration of the request.
=================== ==========================================================
swift-1.13.1/doc/source/db.rst0000664000175400017540000000056312323703611017255 0ustar  jenkinsjenkins00000000000000.. _account_and_container_db:

***************************
Account DB and Container DB
***************************

.. _db:

DB
==

.. automodule:: swift.common.db
    :members:
    :undoc-members:
    :show-inheritance:

.. _db-replicator:

DB replicator
=============

.. automodule:: swift.common.db_replicator
    :members:
    :undoc-members:
    :show-inheritance:
swift-1.13.1/doc/source/crossdomain.rst0000664000175400017540000000371512323703611021213 0ustar  jenkinsjenkins00000000000000========================
Cross-domain Policy File
========================

A cross-domain policy file allows web pages hosted elsewhere to use client
side technologies such as Flash, Java and Silverlight to interact
with the Swift API.

See http://www.adobe.com/devnet/articles/crossdomain_policy_file_spec.html for
a description of the purpose and structure of the cross-domain policy
file. The cross-domain policy file is installed in the root of a web
server (i.e., the path is /crossdomain.xml).

The crossdomain middleware responds to a path of /crossdomain.xml with an
XML document such as::

    
    
    
        
    

You should use a policy appropriate to your site. The examples and the
default policy are provided to indicate how to syntactically construct
a cross domain policy file -- they are not recommendations.

-------------
Configuration
-------------

To enable this middleware, add it to the pipeline in your proxy-server.conf
file. It should be added before any authentication (e.g., tempauth or
keystone) middleware. In this example ellipsis (...) indicate other
middleware you may have chosen to use::

    [pipeline:main]
    pipeline =  ... crossdomain ... authtoken ... proxy-server

And add a filter section, such as::

    [filter:crossdomain]
    use = egg:swift#crossdomain
    cross_domain_policy = 
        

For continuation lines, put some whitespace before the continuation
text. Ensure you put a completely blank line to terminate the
cross_domain_policy value.

The cross_domain_policy name/value is optional. If omitted, the policy
defaults as if you had specified::

    cross_domain_policy = 


swift-1.13.1/doc/source/overview_auth.rst0000664000175400017540000002336012323703611021557 0ustar  jenkinsjenkins00000000000000===============
The Auth System
===============

--------
TempAuth
--------

The auth system for Swift is loosely based on the auth system from the existing
Rackspace architecture -- actually from a few existing auth systems -- and is
therefore a bit disjointed. The distilled points about it are:

* The authentication/authorization part can be an external system or a
  subsystem run within Swift as WSGI middleware
* The user of Swift passes in an auth token with each request
* Swift validates each token with the external auth system or auth subsystem
  and caches the result
* The token does not change from request to request, but does expire

The token can be passed into Swift using the X-Auth-Token or the
X-Storage-Token header. Both have the same format: just a simple string
representing the token. Some auth systems use UUID tokens, some an MD5 hash of
something unique, some use "something else" but the salient point is that the
token is a string which can be sent as-is back to the auth system for
validation.

Swift will make calls to the auth system, giving the auth token to be
validated. For a valid token, the auth system responds with an overall
expiration in seconds from now. Swift will cache the token up to the expiration
time.

The included TempAuth also has the concept of admin and non-admin users
within an account.  Admin users can do anything within the account.
Non-admin users can only perform operations per container based on the
container's X-Container-Read and X-Container-Write ACLs.  Container ACLs
use the "V1" ACL syntax, which looks like this:
``name1, name2, .r:referrer1.com, .r:-bad.referrer1.com, .rlistings``
For more information on ACLs, see :mod:`swift.common.middleware.acl`.

Additionally, if the auth system sets the request environ's swift_owner key to
True, the proxy will return additional header information in some requests,
such as the X-Container-Sync-Key for a container GET or HEAD.

In addition to container ACLs, TempAuth allows account-level ACLs.  Any auth
system may use the special header ``X-Account-Access-Control`` to specify
account-level ACLs in a format specific to that auth system.  (Following the
TempAuth format is strongly recommended.)  These headers are visible and
settable only by account owners (those for whom ``swift_owner`` is true).
Behavior of account ACLs is auth-system-dependent.  In the case of TempAuth,
if an authenticated user has membership in a group which is listed in the
ACL, then the user is allowed the access level of that ACL.

Account ACLs use the "V2" ACL syntax, which is a JSON dictionary with keys
named "admin", "read-write", and "read-only".  (Note the case sensitivity.)
An example value for the ``X-Account-Access-Control`` header looks like this:
``{"admin":["a","b"],"read-only":["c"]}``  Keys may be absent (as shown).
The recommended way to generate ACL strings is as follows::

  from swift.common.middleware.acl import format_acl
  acl_data = { 'admin': ['alice'], 'read-write': ['bob', 'carol'] }
  acl_string = format_acl(version=2, acl_dict=acl_data)

Using the :func:`format_acl` method will ensure
that JSON is encoded as ASCII (using e.g. '\u1234' for Unicode).  While
it's permissible to manually send ``curl`` commands containing
``X-Account-Access-Control`` headers, you should exercise caution when
doing so, due to the potential for human error.

Within the JSON dictionary stored in ``X-Account-Access-Control``, the keys
have the following meanings:

============   ==============================================================
Access Level   Description
============   ==============================================================
read-only      These identities can read *everything* (except privileged
               headers) in the account.  Specifically, a user with read-only
               account access can get a list of containers in the account,
               list the contents of any container, retrieve any object, and
               see the (non-privileged) headers of the account, any
               container, or any object.
read-write     These identities can read or write (or create) any container.
               A user with read-write account access can create new
               containers, set any unprivileged container headers, overwrite
               objects, delete containers, etc.  A read-write user can NOT
               set account headers (or perform any PUT/POST/DELETE requests
               on the account).
admin          These identities have "swift_owner" privileges.  A user with
               admin account access can do anything the account owner can,
               including setting account headers and any privileged headers
               -- and thus granting read-only, read-write, or admin access
               to other users.
============   ==============================================================


For more details, see :mod:`swift.common.middleware.tempauth`.  For details
on the ACL format, see :mod:`swift.common.middleware.acl`.

Users with the special group ``.reseller_admin`` can operate on any account.
For an example usage please see :mod:`swift.common.middleware.tempauth`.
If a request is coming from a reseller the auth system sets the request environ
reseller_request to True. This can be used by other middlewares.

TempAuth will now allow OPTIONS requests to go through without a token.

The user starts a session by sending a ReST request to the auth system to
receive the auth token and a URL to the Swift system.

-------------
Keystone Auth
-------------

Swift is able to authenticate against OpenStack keystone via the
:mod:`swift.common.middleware.keystoneauth` middleware.

In order to use the ``keystoneauth`` middleware the ``authtoken``
middleware from python-keystoneclient will need to be configured.

The ``authtoken`` middleware performs the authentication token
validation and retrieves actual user authentication information. It
can be found in the python-keystoneclient distribution.

The ``keystoneauth`` middleware performs authorization and mapping the
``keystone`` roles to Swift's ACLs.

Configuring Swift to use Keystone
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Configuring Swift to use Keystone is relatively straight
forward.  The first step is to ensure that you have the auth_token
middleware installed, distributed with keystone it can either be
dropped in your python path or installed via the keystone package.

You need at first make sure you have a service endpoint of type
``object-store`` in keystone pointing to your Swift proxy. For example
having this in your ``/etc/keystone/default_catalog.templates`` ::

  catalog.RegionOne.object_store.name = Swift Service
  catalog.RegionOne.object_store.publicURL = http://swiftproxy:8080/v1/AUTH_$(tenant_id)s
  catalog.RegionOne.object_store.adminURL = http://swiftproxy:8080/
  catalog.RegionOne.object_store.internalURL = http://swiftproxy:8080/v1/AUTH_$(tenant_id)s

On your Swift Proxy server you will want to adjust your main pipeline
and add auth_token and keystoneauth in your
``/etc/swift/proxy-server.conf`` like this ::

  [pipeline:main]
  pipeline = [....] authtoken keystoneauth proxy-logging proxy-server

add the configuration for the authtoken middleware::

  [filter:authtoken]
  paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
  auth_host = keystonehost
  auth_port = 35357
  auth_protocol = http
  auth_uri = http://keystonehost:5000/
  admin_tenant_name = service
  admin_user = swift
  admin_password = password
  cache = swift.cache
  include_service_catalog = False

The actual values for these variables will need to be set depending on
your situation.  For more information, please refer to the Keystone
documentation on the ``auth_token`` middleware, but in short:

* Those variables beginning with ``auth_`` point to the Keystone
  Admin service.  This information is used by the middleware to actually
  query Keystone about the validity of the
  authentication tokens.
* The admin auth credentials (``admin_user``, ``admin_tenant_name``,
  ``admin_password``) will be used to retrieve an admin token. That
  token will be used to authorize user tokens behind the scenes.
* cache is set to ``swift.cache``. This means that the middleware
  will get the Swift memcache from the request environment.
* include_service_catalog defaults to True if not set. This means
  that when validating a token, the service catalog is retrieved
  and stored in the X-Service-Catalog header. Since Swift does not
  use the X-Service-Catalog header, there is no point in getting
  the service catalog. We recommend you set include_service_catalog
  to False.


.. note::

    If support is required for unvalidated users (as with anonymous
    access) or for tempurl/formpost middleware, authtoken will need
    to be configured with delay_auth_decision set to 1.

and you can finally add the keystoneauth configuration::

  [filter:keystoneauth]
  use = egg:swift#keystoneauth
  operator_roles = admin, swiftoperator

By default the only users able to give ACL or to Create other
containers are the ones who has the Keystone role specified in the
``operator_roles`` setting.

This user who have one of those role will be able to give ACLs to
other users on containers, see the documentation on ACL here
:mod:`swift.common.middleware.acl`.

Users with the Keystone role defined in ``reseller_admin_role``
(``ResellerAdmin`` by default) can operate on any account. The auth system
sets the request environ reseller_request to True if a request is coming
from an user with this role. This can be used by other middlewares.

--------------
Extending Auth
--------------

TempAuth is written as wsgi middleware, so implementing your own auth is as
easy as writing new wsgi middleware, and plugging it in to the proxy server.
The KeyStone project and the Swauth project are examples of additional auth
services.

Also, see :doc:`development_auth`.
swift-1.13.1/doc/source/admin_guide.rst0000664000175400017540000017427312323703611021147 0ustar  jenkinsjenkins00000000000000=====================
Administrator's Guide
=====================

------------------
Managing the Rings
------------------

You may build the storage rings on any server with the appropriate
version of Swift installed.  Once built or changed (rebalanced), you
must distribute the rings to all the servers in the cluster.  Storage
rings contain information about all the Swift storage partitions and
how they are distributed between the different nodes and disks.

Swift 1.6.0 is the last version to use a Python pickle format.
Subsequent versions use a different serialization format.  **Rings
generated by Swift versions 1.6.0 and earlier may be read by any
version, but rings generated after 1.6.0 may only be read by Swift
versions greater than 1.6.0.**  So when upgrading from version 1.6.0 or
earlier to a version greater than 1.6.0, either upgrade Swift on your
ring building server **last** after all Swift nodes have been successfully
upgraded, or refrain from generating rings until all Swift nodes have
been successfully upgraded.

If you need to downgrade from a version of swift greater than 1.6.0 to
a version less than or equal to 1.6.0, first downgrade your ring-building
server, generate new rings, push them out, then continue with the rest
of the downgrade.

For more information see :doc:`overview_ring`.

Removing a device from the ring::

    swift-ring-builder  remove /
    
Removing a server from the ring::

    swift-ring-builder  remove 
    
Adding devices to the ring:

See :ref:`ring-preparing`
    
See what devices for a server are in the ring::

    swift-ring-builder  search 

Once you are done with all changes to the ring, the changes need to be
"committed"::

    swift-ring-builder  rebalance
    
Once the new rings are built, they should be pushed out to all the servers
in the cluster.

Optionally, if invoked as 'swift-ring-builder-safe' the directory containing
the specified builder file will be locked (via a .lock file in the parent
directory). This provides a basic safe guard against multiple instances
of the swift-ring-builder (or other utilities that observe this lock) from
attempting to write to or read the builder/ring files while operations are in
progress. This can be useful in environments where ring management has been
automated but the operator still needs to interact with the rings manually.

-----------------------
Scripting Ring Creation
-----------------------
You can create scripts to create the account and container rings and rebalance. Here's an example script for the Account ring. Use similar commands to create a make-container-ring.sh script on the proxy server node.

1. Create a script file called make-account-ring.sh on the proxy
   server node with the following content::

    #!/bin/bash
    cd /etc/swift
    rm -f account.builder account.ring.gz backups/account.builder backups/account.ring.gz
    swift-ring-builder account.builder create 18 3 1
    swift-ring-builder account.builder add z1-:6002/sdb1 1
    swift-ring-builder account.builder add z2-:6002/sdb1 1
    swift-ring-builder account.builder rebalance

   You need to replace the values of ,
   , etc. with the IP addresses of the account
   servers used in your setup. You can have as many account servers as
   you need. All account servers are assumed to be listening on port
   6002, and have a storage device called "sdb1" (this is a directory
   name created under /drives when we setup the account server). The
   "z1", "z2", etc. designate zones, and you can choose whether you
   put devices in the same or different zones.

2. Make the script file executable and run it to create the account ring file::

    chmod +x make-account-ring.sh
    sudo ./make-account-ring.sh

3. Copy the resulting ring file /etc/swift/account.ring.gz to all the
   account server nodes in your Swift environment, and put them in the
   /etc/swift directory on these nodes. Make sure that every time you
   change the account ring configuration, you copy the resulting ring
   file to all the account nodes.

-----------------------
Handling System Updates
-----------------------

It is recommended that system updates and reboots are done a zone at a time.
This allows the update to happen, and for the Swift cluster to stay available
and responsive to requests.  It is also advisable when updating a zone, let
it run for a while before updating the other zones to make sure the update
doesn't have any adverse effects.

----------------------
Handling Drive Failure
----------------------

In the event that a drive has failed, the first step is to make sure the drive
is unmounted.  This will make it easier for swift to work around the failure
until it has been resolved.  If the drive is going to be replaced immediately,
then it is just best to replace the drive, format it, remount it, and let
replication fill it up.

If the drive can't be replaced immediately, then it is best to leave it
unmounted, and remove the drive from the ring. This will allow all the
replicas that were on that drive to be replicated elsewhere until the drive
is replaced.  Once the drive is replaced, it can be re-added to the ring.

-----------------------
Handling Server Failure
-----------------------

If a server is having hardware issues, it is a good idea to make sure the 
swift services are not running.  This will allow Swift to work around the
failure while you troubleshoot.

If the server just needs a reboot, or a small amount of work that should
only last a couple of hours, then it is probably best to let Swift work
around the failure and get the machine fixed and back online.  When the
machine comes back online, replication will make sure that anything that is
missing during the downtime will get updated.

If the server has more serious issues, then it is probably best to remove
all of the server's devices from the ring.  Once the server has been repaired
and is back online, the server's devices can be added back into the ring.
It is important that the devices are reformatted before putting them back
into the ring as it is likely to be responsible for a different set of
partitions than before.

-----------------------
Detecting Failed Drives
-----------------------

It has been our experience that when a drive is about to fail, error messages
will spew into `/var/log/kern.log`.  There is a script called
`swift-drive-audit` that can be run via cron to watch for bad drives.  If 
errors are detected, it will unmount the bad drive, so that Swift can
work around it.  The script takes a configuration file with the following
settings:

[drive-audit]

==================  ==============  ===========================================
Option              Default         Description
------------------  --------------  -------------------------------------------
log_facility        LOG_LOCAL0      Syslog log facility
log_level           INFO            Log level
device_dir          /srv/node       Directory devices are mounted under
minutes             60              Number of minutes to look back in
                                    `/var/log/kern.log`
error_limit         1               Number of errors to find before a device
                                    is unmounted
log_file_pattern    /var/log/kern*  Location of the log file with globbing
                                    pattern to check against device errors
regex_pattern_X     (see below)     Regular expression patterns to be used to
                                    locate device blocks with errors in the
                                    log file  
==================  ==============  ===========================================

The default regex pattern used to locate device blocks with errors are
`\berror\b.*\b(sd[a-z]{1,2}\d?)\b` and `\b(sd[a-z]{1,2}\d?)\b.*\berror\b`.
One is able to overwrite the default above by providing new expressions
using the format `regex_pattern_X = regex_expression`, where `X` is a number.

This script has been tested on Ubuntu 10.04 and Ubuntu 12.04, so if you are
using a different distro or OS, some care should be taken before using in production.

--------------
Cluster Health
--------------

There is a swift-dispersion-report tool for measuring overall cluster health.
This is accomplished by checking if a set of deliberately distributed
containers and objects are currently in their proper places within the cluster.

For instance, a common deployment has three replicas of each object. The health
of that object can be measured by checking if each replica is in its proper
place. If only 2 of the 3 is in place the object's heath can be said to be at
66.66%, where 100% would be perfect.

A single object's health, especially an older object, usually reflects the
health of that entire partition the object is in. If we make enough objects on
a distinct percentage of the partitions in the cluster, we can get a pretty
valid estimate of the overall cluster health. In practice, about 1% partition
coverage seems to balance well between accuracy and the amount of time it takes
to gather results.

The first thing that needs to be done to provide this health value is create a
new account solely for this usage. Next, we need to place the containers and
objects throughout the system so that they are on distinct partitions. The
swift-dispersion-populate tool does this by making up random container and
object names until they fall on distinct partitions. Last, and repeatedly for
the life of the cluster, we need to run the swift-dispersion-report tool to
check the health of each of these containers and objects.

These tools need direct access to the entire cluster and to the ring files
(installing them on a proxy server will probably do). Both
swift-dispersion-populate and swift-dispersion-report use the same
configuration file, /etc/swift/dispersion.conf. Example conf file::

    [dispersion]
    auth_url = http://localhost:8080/auth/v1.0
    auth_user = test:tester
    auth_key = testing
    endpoint_type = internalURL

There are also options for the conf file for specifying the dispersion coverage
(defaults to 1%), retries, concurrency, etc. though usually the defaults are
fine.

Once the configuration is in place, run `swift-dispersion-populate` to populate
the containers and objects throughout the cluster.

Now that those containers and objects are in place, you can run
`swift-dispersion-report` to get a dispersion report, or the overall health of
the cluster. Here is an example of a cluster in perfect health::

    $ swift-dispersion-report
    Queried 2621 containers for dispersion reporting, 19s, 0 retries
    100.00% of container copies found (7863 of 7863)
    Sample represents 1.00% of the container partition space
    
    Queried 2619 objects for dispersion reporting, 7s, 0 retries
    100.00% of object copies found (7857 of 7857)
    Sample represents 1.00% of the object partition space

Now I'll deliberately double the weight of a device in the object ring (with
replication turned off) and rerun the dispersion report to show what impact
that has::

    $ swift-ring-builder object.builder set_weight d0 200
    $ swift-ring-builder object.builder rebalance
    ...
    $ swift-dispersion-report
    Queried 2621 containers for dispersion reporting, 8s, 0 retries
    100.00% of container copies found (7863 of 7863)
    Sample represents 1.00% of the container partition space
    
    Queried 2619 objects for dispersion reporting, 7s, 0 retries
    There were 1763 partitions missing one copy.
    77.56% of object copies found (6094 of 7857)
    Sample represents 1.00% of the object partition space

You can see the health of the objects in the cluster has gone down
significantly. Of course, I only have four devices in this test environment, in
a production environment with many many devices the impact of one device change
is much less. Next, I'll run the replicators to get everything put back into
place and then rerun the dispersion report::

    ... start object replicators and monitor logs until they're caught up ...
    $ swift-dispersion-report
    Queried 2621 containers for dispersion reporting, 17s, 0 retries
    100.00% of container copies found (7863 of 7863)
    Sample represents 1.00% of the container partition space

    Queried 2619 objects for dispersion reporting, 7s, 0 retries
    100.00% of object copies found (7857 of 7857)
    Sample represents 1.00% of the object partition space

You can also run the report for only containers or objects::

    $ swift-dispersion-report --container-only
    Queried 2621 containers for dispersion reporting, 17s, 0 retries
    100.00% of container copies found (7863 of 7863)
    Sample represents 1.00% of the container partition space

    $ swift-dispersion-report --object-only
    Queried 2619 objects for dispersion reporting, 7s, 0 retries
    100.00% of object copies found (7857 of 7857)
    Sample represents 1.00% of the object partition space

Alternatively, the dispersion report can also be output in json format. This 
allows it to be more easily consumed by third party utilities::

    $ swift-dispersion-report -j
    {"object": {"retries:": 0, "missing_two": 0, "copies_found": 7863, "missing_one": 0, "copies_expected": 7863, "pct_found": 100.0, "overlapping": 0, "missing_all": 0}, "container": {"retries:": 0, "missing_two": 0, "copies_found": 12534, "missing_one": 0, "copies_expected": 12534, "pct_found": 100.0, "overlapping": 15, "missing_all": 0}}


-----------------------------------
Geographically Distributed Clusters
-----------------------------------

Swift's default configuration is currently designed to work in a
single region, where a region is defined as a group of machines with
high-bandwidth, low-latency links between them. However, configuration
options exist that make running a performant multi-region Swift
cluster possible.

For the rest of this section, we will assume a two-region Swift
cluster: region 1 in San Francisco (SF), and region 2 in New York
(NY). Each region shall contain within it 3 zones, numbered 1, 2, and
3, for a total of 6 zones.

~~~~~~~~~~~~~
read_affinity
~~~~~~~~~~~~~

This setting makes the proxy server prefer local backend servers for
GET and HEAD requests over non-local ones. For example, it is
preferable for an SF proxy server to service object GET requests
by talking to SF object servers, as the client will receive lower
latency and higher throughput.

By default, Swift randomly chooses one of the three replicas to give
to the client, thereby spreading the load evenly. In the case of a
geographically-distributed cluster, the administrator is likely to
prioritize keeping traffic local over even distribution of results.
This is where the read_affinity setting comes in.

Example::

    [app:proxy-server]
    read_affinity = r1=100

This will make the proxy attempt to service GET and HEAD requests from
backends in region 1 before contacting any backends in region 2.
However, if no region 1 backends are available (due to replica
placement, failed hardware, or other reasons), then the proxy will
fall back to backend servers in other regions.

Example::

    [app:proxy-server]
    read_affinity = r1z1=100, r1=200

This will make the proxy attempt to service GET and HEAD requests from
backends in region 1 zone 1, then backends in region 1, then any other
backends. If a proxy is physically close to a particular zone or
zones, this can provide bandwidth savings. For example, if a zone
corresponds to servers in a particular rack, and the proxy server is
in that same rack, then setting read_affinity to prefer reads from
within the rack will result in less traffic between the top-of-rack
switches.

The read_affinity setting may contain any number of region/zone
specifiers; the priority number (after the equals sign) determines the
ordering in which backend servers will be contacted. A lower number
means higher priority.

Note that read_affinity only affects the ordering of primary nodes
(see ring docs for definition of primary node), not the ordering of
handoff nodes.

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
write_affinity and write_affinity_node_count
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

This setting makes the proxy server prefer local backend servers for
object PUT requests over non-local ones. For example, it may be
preferable for an SF proxy server to service object PUT requests
by talking to SF object servers, as the client will receive lower
latency and higher throughput. However, if this setting is used, note
that a NY proxy server handling a GET request for an object that was
PUT using write affinity may have to fetch it across the WAN link, as
the object won't immediately have any replicas in NY. However,
replication will move the object's replicas to their proper homes in
both SF and NY.

Note that only object PUT requests are affected by the write_affinity
setting; POST, GET, HEAD, DELETE, OPTIONS, and account/container PUT
requests are not affected.

This setting lets you trade data distribution for throughput. If
write_affinity is enabled, then object replicas will initially be
stored all within a particular region or zone, thereby decreasing the
quality of the data distribution, but the replicas will be distributed
over fast WAN links, giving higher throughput to clients. Note that
the replicators will eventually move objects to their proper,
well-distributed homes.

The write_affinity setting is useful only when you don't typically
read objects immediately after writing them. For example, consider a
workload of mainly backups: if you have a bunch of machines in NY that
periodically write backups to Swift, then odds are that you don't then
immediately read those backups in SF. If your workload doesn't look
like that, then you probably shouldn't use write_affinity.

The write_affinity_node_count setting is only useful in conjunction
with write_affinity; it governs how many local object servers will be
tried before falling back to non-local ones.

Example::

    [app:proxy-server]
    write_affinity = r1
    write_affinity_node_count = 2 * replicas

Assuming 3 replicas, this configuration will make object PUTs try
storing the object's replicas on up to 6 disks ("2 * replicas") in
region 1 ("r1").

You should be aware that, if you have data coming into SF faster than
your link to NY can transfer it, then your cluster's data distribution
will get worse and worse over time as objects pile up in SF. If this
happens, it is recommended to disable write_affinity and simply let
object PUTs traverse the WAN link, as that will naturally limit the
object growth rate to what your WAN link can handle.


--------------------------------
Cluster Telemetry and Monitoring
--------------------------------

Various metrics and telemetry can be obtained from the account, container, and
object servers using the recon server middleware and the swift-recon cli. To do
so update your account, container, or object servers pipelines to include recon
and add the associated filter config.

object-server.conf sample::

    [pipeline:main]
    pipeline = recon object-server

    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift

container-server.conf sample::

    [pipeline:main]
    pipeline = recon container-server

    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift

account-server.conf sample::

    [pipeline:main]
    pipeline = recon account-server

    [filter:recon]
    use = egg:swift#recon
    recon_cache_path = /var/cache/swift

The recon_cache_path simply sets the directory where stats for a few items will
be stored. Depending on the method of deployment you may need to create this
directory manually and ensure that swift has read/write access.

Finally, if you also wish to track asynchronous pending on your object
servers you will need to setup a cronjob to run the swift-recon-cron script
periodically on your object servers::

    */5 * * * * swift /usr/bin/swift-recon-cron /etc/swift/object-server.conf

Once the recon middleware is enabled, a GET request for
"/recon/" to the backend object server will return a
JSON-formatted response::

    fhines@ubuntu:~$ curl -i http://localhost:6030/recon/async
    HTTP/1.1 200 OK
    Content-Type: application/json
    Content-Length: 20
    Date: Tue, 18 Oct 2011 21:03:01 GMT

    {"async_pending": 0}


Note that the default port for the object server is 6000, except on a
Swift All-In-One installation, which uses 6010, 6020, 6030, and 6040.

The following metrics and telemetry are currently exposed:

=========================   ========================================================================================
Request URI                 Description
-------------------------   ----------------------------------------------------------------------------------------
/recon/load                 returns 1,5, and 15 minute load average
/recon/mem                  returns /proc/meminfo
/recon/mounted              returns *ALL* currently mounted filesystems
/recon/unmounted            returns all unmounted drives if mount_check = True
/recon/diskusage            returns disk utilization for storage devices
/recon/ringmd5              returns object/container/account ring md5sums
/recon/quarantined          returns # of quarantined objects/accounts/containers
/recon/sockstat             returns consumable info from /proc/net/sockstat|6
/recon/devices              returns list of devices and devices dir i.e. /srv/node
/recon/async                returns count of async pending
/recon/replication          returns object replication times (for backward compatibility)
/recon/replication/   returns replication info for given type (account, container, object)
/recon/auditor/       returns auditor stats on last reported scan for given type (account, container, object)
/recon/updater/       returns last updater sweep times for given type (container, object)
=========================   ========================================================================================

This information can also be queried via the swift-recon command line utility::

    fhines@ubuntu:~$ swift-recon -h
    Usage: 
            usage: swift-recon  [-v] [--suppress] [-a] [-r] [-u] [-d]
            [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]

               account|container|object
            Defaults to object server.

            ex: swift-recon container -l --auditor


    Options:
      -h, --help            show this help message and exit
      -v, --verbose         Print verbose info
      --suppress            Suppress most connection related errors
      -a, --async           Get async stats
      -r, --replication     Get replication stats
      --auditor             Get auditor stats
      --updater             Get updater stats
      --expirer             Get expirer stats
      -u, --unmounted       Check cluster for unmounted devices
      -d, --diskusage       Get disk usage stats
      -l, --loadstats       Get cluster load average stats
      -q, --quarantined     Get cluster quarantine stats
      --md5                 Get md5sum of servers ring and compare to local copy
      --sockstat            Get cluster socket usage stats
      --all                 Perform all checks. Equal to -arudlq --md5 --sockstat
      -z ZONE, --zone=ZONE  Only query servers in specified zone
      -t SECONDS, --timeout=SECONDS
                            Time to wait for a response from a server
      --swiftdir=SWIFTDIR   Default = /etc/swift

For example, to obtain container replication info from all hosts in zone "3"::

    fhines@ubuntu:~$ swift-recon container -r --zone 3
    ===============================================================================
    --> Starting reconnaissance on 1 hosts
    ===============================================================================
    [2012-04-02 02:45:48] Checking on replication
    [failure] low: 0.000, high: 0.000, avg: 0.000, reported: 1
    [success] low: 486.000, high: 486.000, avg: 486.000, reported: 1
    [replication_time] low: 20.853, high: 20.853, avg: 20.853, reported: 1
    [attempted] low: 243.000, high: 243.000, avg: 243.000, reported: 1

---------------------------
Reporting Metrics to StatsD
---------------------------

If you have a StatsD_ server running, Swift may be configured to send it
real-time operational metrics.  To enable this, set the following
configuration entries (see the sample configuration files)::

    log_statsd_host = localhost
    log_statsd_port = 8125
    log_statsd_default_sample_rate = 1.0
    log_statsd_sample_rate_factor = 1.0
    log_statsd_metric_prefix =                [empty-string]

If `log_statsd_host` is not set, this feature is disabled.  The default values
for the other settings are given above.

.. _StatsD: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
.. _Graphite: http://graphite.wikidot.com/
.. _Ganglia: http://ganglia.sourceforge.net/

The sample rate is a real number between 0 and 1 which defines the
probability of sending a sample for any given event or timing measurement.
This sample rate is sent with each sample to StatsD and used to
multiply the value.  For example, with a sample rate of 0.5, StatsD will
multiply that counter's value by 2 when flushing the metric to an upstream
monitoring system (Graphite_, Ganglia_, etc.).

Some relatively high-frequency metrics have a default sample rate less than
one.  If you want to override the default sample rate for all metrics whose
default sample rate is not specified in the Swift source, you may set
`log_statsd_default_sample_rate` to a value less than one.  This is NOT
recommended (see next paragraph).  A better way to reduce StatsD load is to
adjust `log_statsd_sample_rate_factor` to a value less than one.  The
`log_statsd_sample_rate_factor` is multiplied to any sample rate (either the
global default or one specified by the actual metric logging call in the Swift
source) prior to handling.  In other words, this one tunable can lower the
frequency of all StatsD logging by a proportional amount.

To get the best data, start with the default `log_statsd_default_sample_rate`
and `log_statsd_sample_rate_factor` values of 1 and only lower
`log_statsd_sample_rate_factor` if needed.  The
`log_statsd_default_sample_rate` should not be used and remains for backward
compatibility only.

The metric prefix will be prepended to every metric sent to the StatsD server
For example, with::

    log_statsd_metric_prefix = proxy01

the metric `proxy-server.errors` would be sent to StatsD as
`proxy01.proxy-server.errors`.  This is useful for differentiating different
servers when sending statistics to a central StatsD server.  If you run a local
StatsD server per node, you could configure a per-node metrics prefix there and
leave `log_statsd_metric_prefix` blank.

Note that metrics reported to StatsD are counters or timing data (which are
sent in units of milliseconds).  StatsD usually expands timing data out to min,
max, avg, count, and 90th percentile per timing metric, but the details of
this behavior will depend on the configuration of your StatsD server.  Some
important "gauge" metrics may still need to be collected using another method.
For example, the `object-server.async_pendings` StatsD metric counts the generation
of async_pendings in real-time, but will not tell you the current number of
async_pending container updates on disk at any point in time.

Note also that the set of metrics collected, their names, and their semantics
are not locked down and will change over time.  StatsD logging is currently in
a "beta" stage and will continue to evolve.

Metrics for `account-auditor`:

==========================  =========================================================
Metric Name                 Description
--------------------------  ---------------------------------------------------------
`account-auditor.errors`    Count of audit runs (across all account databases) which
                            caught an Exception.
`account-auditor.passes`    Count of individual account databases which passed audit.
`account-auditor.failures`  Count of individual account databases which failed audit.
`account-auditor.timing`    Timing data for individual account database audits.
==========================  =========================================================

Metrics for `account-reaper`:

==============================================  ====================================================
Metric Name                                     Description
----------------------------------------------  ----------------------------------------------------
`account-reaper.errors`                         Count of devices failing the mount check.
`account-reaper.timing`                         Timing data for each reap_account() call.
`account-reaper.return_codes.X`                 Count of HTTP return codes from various operations
                                                (e.g. object listing, container deletion, etc.). The
                                                value for X is the first digit of the return code
                                                (2 for 201, 4 for 404, etc.).
`account-reaper.containers_failures`            Count of failures to delete a container.
`account-reaper.containers_deleted`             Count of containers successfully deleted.
`account-reaper.containers_remaining`           Count of containers which failed to delete with
                                                zero successes.
`account-reaper.containers_possibly_remaining`  Count of containers which failed to delete with
                                                at least one success.
`account-reaper.objects_failures`               Count of failures to delete an object.
`account-reaper.objects_deleted`                Count of objects successfully deleted.
`account-reaper.objects_remaining`              Count of objects which failed to delete with zero
                                                successes.
`account-reaper.objects_possibly_remaining`     Count of objects which failed to delete with at
                                                least one success.
==============================================  ====================================================

Metrics for `account-server` ("Not Found" is not considered an error and requests
which increment `errors` are not included in the timing data):

========================================  =======================================================
Metric Name                               Description
----------------------------------------  -------------------------------------------------------
`account-server.DELETE.errors.timing`     Timing data for each DELETE request resulting in an
                                          error: bad request, not mounted, missing timestamp.
`account-server.DELETE.timing`            Timing data for each DELETE request not resulting in
                                          an error.
`account-server.PUT.errors.timing`        Timing data for each PUT request resulting in an error:
                                          bad request, not mounted, conflict, recently-deleted.
`account-server.PUT.timing`               Timing data for each PUT request not resulting in an
                                          error.
`account-server.HEAD.errors.timing`       Timing data for each HEAD request resulting in an
                                          error: bad request, not mounted.
`account-server.HEAD.timing`              Timing data for each HEAD request not resulting in
                                          an error.
`account-server.GET.errors.timing`        Timing data for each GET request resulting in an
                                          error: bad request, not mounted, bad delimiter,
                                          account listing limit too high, bad accept header.
`account-server.GET.timing`               Timing data for each GET request not resulting in
                                          an error.
`account-server.REPLICATE.errors.timing`  Timing data for each REPLICATE request resulting in an
                                          error: bad request, not mounted.
`account-server.REPLICATE.timing`         Timing data for each REPLICATE request not resulting
                                          in an error.
`account-server.POST.errors.timing`       Timing data for each POST request resulting in an
                                          error: bad request, bad or missing timestamp, not
                                          mounted.
`account-server.POST.timing`              Timing data for each POST request not resulting in
                                          an error.
========================================  =======================================================

Metrics for `account-replicator`:

=====================================  ====================================================
Metric Name                            Description
-------------------------------------  ----------------------------------------------------
`account-replicator.diffs`             Count of syncs handled by sending differing rows.
`account-replicator.diff_caps`         Count of "diffs" operations which failed because
                                       "max_diffs" was hit.
`account-replicator.no_changes`        Count of accounts found to be in sync.
`account-replicator.hashmatches`       Count of accounts found to be in sync via hash
                                       comparison (`broker.merge_syncs` was called).
`account-replicator.rsyncs`            Count of completely missing accounts which were sent
                                       via rsync.
`account-replicator.remote_merges`     Count of syncs handled by sending entire database
                                       via rsync.
`account-replicator.attempts`          Count of database replication attempts.
`account-replicator.failures`          Count of database replication attempts which failed
                                       due to corruption (quarantined) or inability to read
                                       as well as attempts to individual nodes which
                                       failed.
`account-replicator.removes.`  Count of databases on  deleted because the
                                       delete_timestamp was greater than the put_timestamp
                                       and the database had no rows or because it was
                                       successfully sync'ed to other locations and doesn't
                                       belong here anymore.
`account-replicator.successes`         Count of replication attempts to an individual node
                                       which were successful.
`account-replicator.timing`            Timing data for each database replication attempt
                                       not resulting in a failure.
=====================================  ====================================================

Metrics for `container-auditor`:

============================  ====================================================
Metric Name                   Description
----------------------------  ----------------------------------------------------
`container-auditor.errors`    Incremented when an Exception is caught in an audit
                              pass (only once per pass, max).
`container-auditor.passes`    Count of individual containers passing an audit.
`container-auditor.failures`  Count of individual containers failing an audit.
`container-auditor.timing`    Timing data for each container audit.
============================  ====================================================

Metrics for `container-replicator`:

=======================================  ====================================================
Metric Name                              Description
---------------------------------------  ----------------------------------------------------
`container-replicator.diffs`             Count of syncs handled by sending differing rows.
`container-replicator.diff_caps`         Count of "diffs" operations which failed because
                                         "max_diffs" was hit.
`container-replicator.no_changes`        Count of containers found to be in sync.
`container-replicator.hashmatches`       Count of containers found to be in sync via hash
                                         comparison (`broker.merge_syncs` was called).
`container-replicator.rsyncs`            Count of completely missing containers where were sent
                                         via rsync.
`container-replicator.remote_merges`     Count of syncs handled by sending entire database
                                         via rsync.
`container-replicator.attempts`          Count of database replication attempts.
`container-replicator.failures`          Count of database replication attempts which failed
                                         due to corruption (quarantined) or inability to read
                                         as well as attempts to individual nodes which
                                         failed.
`container-replicator.removes.`  Count of databases deleted on  because the
                                         delete_timestamp was greater than the put_timestamp
                                         and the database had no rows or because it was
                                         successfully sync'ed to other locations and doesn't
                                         belong here anymore.
`container-replicator.successes`         Count of replication attempts to an individual node
                                         which were successful.
`container-replicator.timing`            Timing data for each database replication attempt
                                         not resulting in a failure.
=======================================  ====================================================

Metrics for `container-server` ("Not Found" is not considered an error and requests
which increment `errors` are not included in the timing data):

==========================================  ====================================================
Metric Name                                 Description
------------------------------------------  ----------------------------------------------------
`container-server.DELETE.errors.timing`     Timing data for DELETE request errors: bad request,
                                            not mounted, missing timestamp, conflict.
`container-server.DELETE.timing`            Timing data for each DELETE request not resulting in
                                            an error.
`container-server.PUT.errors.timing`        Timing data for PUT request errors: bad request,
                                            missing timestamp, not mounted, conflict.
`container-server.PUT.timing`               Timing data for each PUT request not resulting in an
                                            error.
`container-server.HEAD.errors.timing`       Timing data for HEAD request errors: bad request,
                                            not mounted.
`container-server.HEAD.timing`              Timing data for each HEAD request not resulting in
                                            an error.
`container-server.GET.errors.timing`        Timing data for GET request errors: bad request,
                                            not mounted, parameters not utf8, bad accept header.
`container-server.GET.timing`               Timing data for each GET request not resulting in
                                            an error.
`container-server.REPLICATE.errors.timing`  Timing data for REPLICATE request errors: bad
                                            request, not mounted.
`container-server.REPLICATE.timing`         Timing data for each REPLICATE request not resulting
                                            in an error.
`container-server.POST.errors.timing`       Timing data for POST request errors: bad request,
                                            bad x-container-sync-to, not mounted.
`container-server.POST.timing`              Timing data for each POST request not resulting in
                                            an error.
==========================================  ====================================================

Metrics for `container-sync`:

===============================  ====================================================
Metric Name                      Description
-------------------------------  ----------------------------------------------------
`container-sync.skips`           Count of containers skipped because they don't have
                                 sync'ing enabled.
`container-sync.failures`        Count of failures sync'ing of individual containers.
`container-sync.syncs`           Count of individual containers sync'ed successfully.
`container-sync.deletes`         Count of container database rows sync'ed by
                                 deletion.
`container-sync.deletes.timing`  Timing data for each container database row
                                 sychronization via deletion.
`container-sync.puts`            Count of container database rows sync'ed by PUTing.
`container-sync.puts.timing`     Timing data for each container database row
                                 synchronization via PUTing.
===============================  ====================================================

Metrics for `container-updater`:

==============================  ====================================================
Metric Name                     Description
------------------------------  ----------------------------------------------------
`container-updater.successes`   Count of containers which successfully updated their
                                account.
`container-updater.failures`    Count of containers which failed to update their
                                account.
`container-updater.no_changes`  Count of containers which didn't need to update
                                their account.
`container-updater.timing`      Timing data for processing a container; only
                                includes timing for containers which needed to
                                update their accounts (i.e. "successes" and
                                "failures" but not "no_changes").
==============================  ====================================================

Metrics for `object-auditor`:

============================  ====================================================
Metric Name                   Description
----------------------------  ----------------------------------------------------
`object-auditor.quarantines`  Count of objects failing audit and quarantined.
`object-auditor.errors`       Count of errors encountered while auditing objects.
`object-auditor.timing`       Timing data for each object audit (does not include
                              any rate-limiting sleep time for
                              max_files_per_second, but does include rate-limiting
                              sleep time for max_bytes_per_second).
============================  ====================================================

Metrics for `object-expirer`:

========================  ====================================================
Metric Name               Description
------------------------  ----------------------------------------------------
`object-expirer.objects`  Count of objects expired.
`object-expirer.errors`   Count of errors encountered while attempting to
                          expire an object.
`object-expirer.timing`   Timing data for each object expiration attempt,
                          including ones resulting in an error.
========================  ====================================================

Metrics for `object-replicator`:

===================================================  ====================================================
Metric Name                                          Description
---------------------------------------------------  ----------------------------------------------------
`object-replicator.partition.delete.count.`  A count of partitions on  which were
                                                     replicated to another node because they didn't
                                                     belong on this node.  This metric is tracked
                                                     per-device to allow for "quiescence detection" for
                                                     object replication activity on each device.
`object-replicator.partition.delete.timing`          Timing data for partitions replicated to another
                                                     node because they didn't belong on this node.  This
                                                     metric is not tracked per device.
`object-replicator.partition.update.count.`  A count of partitions on  which were
                                                     replicated to another node, but also belong on this
                                                     node.  As with delete.count, this metric is tracked
                                                     per-device.
`object-replicator.partition.update.timing`          Timing data for partitions replicated which also
                                                     belong on this node.  This metric is not tracked
                                                     per-device.
`object-replicator.suffix.hashes`                    Count of suffix directories whose hash (of filenames)
                                                     was recalculated.
`object-replicator.suffix.syncs`                     Count of suffix directories replicated with rsync.
===================================================  ====================================================

Metrics for `object-server`:

=======================================  ====================================================
Metric Name                              Description
---------------------------------------  ----------------------------------------------------
`object-server.quarantines`              Count of objects (files) found bad and moved to
                                         quarantine.
`object-server.async_pendings`           Count of container updates saved as async_pendings
                                         (may result from PUT or DELETE requests).
`object-server.POST.errors.timing`       Timing data for POST request errors: bad request,
                                         missing timestamp, delete-at in past, not mounted.
`object-server.POST.timing`              Timing data for each POST request not resulting in
                                         an error.
`object-server.PUT.errors.timing`        Timing data for PUT request errors: bad request,
                                         not mounted, missing timestamp, object creation
                                         constraint violation, delete-at in past.
`object-server.PUT.timeouts`             Count of object PUTs which exceeded max_upload_time.
`object-server.PUT.timing`               Timing data for each PUT request not resulting in an
                                         error.
`object-server.PUT..timing`      Timing data per kB transferred (ms/kB) for each
                                         non-zero-byte PUT request on each device. 
                                         Monitoring problematic devices, higher is bad. 
`object-server.GET.errors.timing`        Timing data for GET request errors: bad request,
                                         not mounted, header timestamps before the epoch,
                                         precondition failed.
                                         File errors resulting in a quarantine are not
                                         counted here.
`object-server.GET.timing`               Timing data for each GET request not resulting in an
                                         error.  Includes requests which couldn't find the
                                         object (including disk errors resulting in file
                                         quarantine).
`object-server.HEAD.errors.timing`       Timing data for HEAD request errors: bad request,
                                         not mounted.
`object-server.HEAD.timing`              Timing data for each HEAD request not resulting in
                                         an error.  Includes requests which couldn't find the
                                         object (including disk errors resulting in file
                                         quarantine).
`object-server.DELETE.errors.timing`     Timing data for DELETE request errors: bad request,
                                         missing timestamp, not mounted, precondition
                                         failed.  Includes requests which couldn't find or
                                         match the object.
`object-server.DELETE.timing`            Timing data for each DELETE request not resulting
                                         in an error.
`object-server.REPLICATE.errors.timing`  Timing data for REPLICATE request errors: bad
                                         request, not mounted.
`object-server.REPLICATE.timing`         Timing data for each REPLICATE request not resulting
                                         in an error.
=======================================  ====================================================

Metrics for `object-updater`:

============================  ====================================================
Metric Name                   Description
----------------------------  ----------------------------------------------------
`object-updater.errors`       Count of drives not mounted or async_pending files
                              with an unexpected name.
`object-updater.timing`       Timing data for object sweeps to flush async_pending
                              container updates.  Does not include object sweeps
                              which did not find an existing async_pending storage
                              directory.
`object-updater.quarantines`  Count of async_pending container updates which were
                              corrupted and moved to quarantine.
`object-updater.successes`    Count of successful container updates.
`object-updater.failures`     Count of failed container updates.
`object-updater.unlinks`      Count of async_pending files unlinked. An
                              async_pending file is unlinked either when it is
                              successfully processed or when the replicator sees
                              that there is a newer async_pending file for the
                              same object.
============================  ====================================================

Metrics for `proxy-server` (in the table, `` is the proxy-server
controller responsible for the request and will be one of "account",
"container", or "object"):

========================================  ====================================================
Metric Name                               Description
----------------------------------------  ----------------------------------------------------
`proxy-server.errors`                     Count of errors encountered while serving requests
                                          before the controller type is determined.  Includes
                                          invalid Content-Length, errors finding the internal
                                          controller to handle the request, invalid utf8, and
                                          bad URLs.
`proxy-server..handoff_count`       Count of node hand-offs; only tracked if log_handoffs
                                          is set in the proxy-server config.
`proxy-server..handoff_all_count`   Count of times *only* hand-off locations were
                                          utilized; only tracked if log_handoffs is set in the
                                          proxy-server config.
`proxy-server..client_timeouts`     Count of client timeouts (client did not read within
                                          `client_timeout` seconds during a GET or did not
                                          supply data within `client_timeout` seconds during
                                          a PUT).
`proxy-server..client_disconnects`  Count of detected client disconnects during PUT
                                          operations (does NOT include caught Exceptions in
                                          the proxy-server which caused a client disconnect).
========================================  ====================================================

Metrics for `proxy-logging` middleware (in the table, `` is either the
proxy-server controller responsible for the request: "account", "container",
"object", or the string "SOS" if the request came from the `Swift Origin Server`_
middleware.  The `` portion will be one of "GET", "HEAD", "POST", "PUT",
"DELETE", "COPY", "OPTIONS", or "BAD_METHOD".  The list of valid HTTP methods
is configurable via the `log_statsd_valid_http_methods` config variable and
the default setting yields the above behavior.

.. _Swift Origin Server: https://github.com/dpgoetz/sos

====================================================  ============================================
Metric Name                                           Description
----------------------------------------------------  --------------------------------------------
`proxy-server....timing`          Timing data for requests, start to finish.
                                                      The  portion is the numeric HTTP
                                                      status code for the request (e.g.  "200" or
                                                      "404").
`proxy-server..GET..first-byte.timing`  Timing data up to completion of sending the
                                                      response headers (only for GET requests).
                                                       and  are as for the main
                                                      timing metric.
`proxy-server....xfer`            This counter metric is the sum of bytes
                                                      transferred in (from clients) and out (to
                                                      clients) for requests.  The , ,
                                                      and  portions of the metric are just
                                                      like the main timing metric.
====================================================  ============================================

Metrics for `tempauth` middleware (in the table, `` represents
the actual configured reseller_prefix or "`NONE`" if the reseller_prefix is the
empty string):

=========================================  ====================================================
Metric Name                                Description
-----------------------------------------  ----------------------------------------------------
`tempauth..unauthorized`  Count of regular requests which were denied with
                                           HTTPUnauthorized.
`tempauth..forbidden`     Count of regular requests which were denied with
                                           HTTPForbidden.
`tempauth..token_denied`  Count of token requests which were denied.
`tempauth..errors`        Count of errors.
=========================================  ====================================================


------------------------
Debugging Tips and Tools
------------------------

When a request is made to Swift, it is given a unique transaction id.  This
id should be in every log line that has to do with that request.  This can
be useful when looking at all the services that are hit by a single request.

If you need to know where a specific account, container or object is in the
cluster, `swift-get-nodes` will show the location where each replica should be.

If you are looking at an object on the server and need more info,
`swift-object-info` will display the account, container, replica locations
and metadata of the object.

If you are looking at a container on the server and need more info,
`swift-container-info` will display all the information like the account,
container, replica locations and metadata of the container.

If you are looking at an account on the server and need more info,
`swift-account-info` will display the account, replica locations
and metadata of the account.

If you want to audit the data for an account, `swift-account-audit` can be
used to crawl the account, checking that all containers and objects can be
found.

-----------------
Managing Services
-----------------

Swift services are generally managed with `swift-init`. the general usage is
``swift-init  ``, where service is the swift service to 
manage (for example object, container, account, proxy) and command is one of:

==========  ===============================================
Command     Description
----------  -----------------------------------------------
start       Start the service
stop        Stop the service
restart     Restart the service
shutdown    Attempt to gracefully shutdown the service
reload      Attempt to gracefully restart the service
==========  ===============================================

A graceful shutdown or reload will finish any current requests before 
completely stopping the old service.  There is also a special case of 
`swift-init all `, which will run the command for all swift services.

--------------
Object Auditor
--------------

On system failures, the XFS file system can sometimes truncate files it's
trying to write and produce zero-byte files. The object-auditor will catch
these problems but in the case of a system crash it would be advisable to run
an extra, less rate limited sweep to check for these specific files. You can
run this command as follows:
`swift-object-auditor /path/to/object-server/config/file.conf once -z 1000`
"-z" means to only check for zero-byte files at 1000 files per second.

At times it is useful to be able to run the object auditor on a specific
device or set of devices.  You can run the object-auditor as follows:
swift-object-auditor /path/to/object-server/config/file.conf once --devices=sda,sdb

This will run the object auditor on only the sda and sdb devices. This param
accepts a comma separated list of values.

-----------------
Object Replicator
-----------------

At times it is useful to be able to run the object replicator on a specific
device or partition.  You can run the object-replicator as follows:
swift-object-replicator /path/to/object-server/config/file.conf once --devices=sda,sdb

This will run the object replicator on only the sda and sdb devices.  You can
likewise run that command with --partitions.  Both params accept a comma
separated list of values. If both are specified they will be ANDed together.
These can only be run in "once" mode.

-------------
Swift Orphans
-------------

Swift Orphans are processes left over after a reload of a Swift server.

For example, when upgrading a proxy server you would probaby finish
with a `swift-init proxy-server reload` or `/etc/init.d/swift-proxy
reload`. This kills the parent proxy server process and leaves the
child processes running to finish processing whatever requests they
might be handling at the time. It then starts up a new parent proxy
server process and its children to handle new incoming requests. This
allows zero-downtime upgrades with no impact to existing requests.

The orphaned child processes may take a while to exit, depending on
the length of the requests they were handling. However, sometimes an
old process can be hung up due to some bug or hardware issue. In these
cases, these orphaned processes will hang around
forever. `swift-orphans` can be used to find and kill these orphans.

`swift-orphans` with no arguments will just list the orphans it finds
that were started more than 24 hours ago. You shouldn't really check
for orphans until 24 hours after you perform a reload, as some
requests can take a long time to process. `swift-orphans -k TERM` will
send the SIG_TERM signal to the orphans processes, or you can `kill
-TERM` the pids yourself if you prefer.

You can run `swift-orphans --help` for more options.


------------
Swift Oldies
------------

Swift Oldies are processes that have just been around for a long
time. There's nothing necessarily wrong with this, but it might
indicate a hung process if you regularly upgrade and reload/restart
services. You might have so many servers that you don't notice when a
reload/restart fails; `swift-oldies` can help with this.

For example, if you upgraded and reloaded/restarted everything 2 days
ago, and you've already cleaned up any orphans with `swift-orphans`,
you can run `swift-oldies -a 48` to find any Swift processes still
around that were started more than 2 days ago and then investigate
them accordingly.



-------------------
Custom Log Handlers
-------------------

Swift supports setting up custom log handlers for services by specifying a
comma-separated list of functions to invoke when logging is setup. It does so
via the `log_custom_handlers` configuration option. Logger hooks invoked are
passed the same arguments as Swift's get_logger function (as well as the
getLogger and LogAdapter object):

==============  ===============================================
Name            Description
--------------  -----------------------------------------------
conf            Configuration dict to read settings from
name            Name of the logger received
log_to_console  (optional) Write log messages to console on stderr
log_route       Route for the logging received
fmt             Override log format received
logger          The logging.getLogger object
adapted_logger  The LogAdapter object
==============  ===============================================

A basic example that sets up a custom logger might look like the
following:


.. code-block:: python

    def my_logger(conf, name, log_to_console, log_route, fmt, logger,
                  adapted_logger):
        my_conf_opt = conf.get('some_custom_setting')
        my_handler = third_party_logstore_handler(my_conf_opt)
        logger.addHandler(my_handler)

See :ref:`custom-logger-hooks-label` for sample use cases.

swift-1.13.1/doc/source/ring.rst0000664000175400017540000000061212323703611017622 0ustar  jenkinsjenkins00000000000000.. _consistent_hashing_ring:

********************************
Partitioned Consistent Hash Ring
********************************

.. _ring:

Ring
====

.. automodule:: swift.common.ring.ring
    :members:
    :undoc-members:
    :show-inheritance:

.. _ring-builder:

Ring Builder
============

.. automodule:: swift.common.ring.builder
    :members:
    :undoc-members:
    :show-inheritance:
swift-1.13.1/doc/source/overview_architecture.rst0000664000175400017540000001436112323703611023301 0ustar  jenkinsjenkins00000000000000============================
Swift Architectural Overview
============================

.. TODO - add links to more detailed overview in each section below.

------------
Proxy Server
------------

The Proxy Server is responsible for tying together the rest of the Swift
architecture. For each request, it will look up the location of the account,
container, or object in the ring (see below) and route the request accordingly.
The public API is also exposed through the Proxy Server.

A large number of failures are also handled in the Proxy Server. For
example, if a server is unavailable for an object PUT, it will ask the
ring for a handoff server and route there instead.

When objects are streamed to or from an object server, they are streamed
directly through the proxy server to or from the user -- the proxy server
does not spool them.

--------
The Ring
--------

A ring represents a mapping between the names of entities stored on disk and
their physical location. There are separate rings for accounts, containers, and
objects. When other components need to perform any operation on an object,
container, or account, they need to interact with the appropriate ring to
determine its location in the cluster.

The Ring maintains this mapping using zones, devices, partitions, and replicas.
Each partition in the ring is replicated, by default, 3 times across the
cluster, and the locations for a partition are stored in the mapping maintained
by the ring. The ring is also responsible for determining which devices are
used for handoff in failure scenarios.

Data can be isolated with the concept of zones in the ring. Each replica
of a partition is guaranteed to reside in a different zone. A zone could
represent a drive, a server, a cabinet, a switch, or even a datacenter.

The partitions of the ring are equally divided among all the devices in the
Swift installation. When partitions need to be moved around (for example if a
device is added to the cluster), the ring ensures that a minimum number of
partitions are moved at a time, and only one replica of a partition is moved at
a time.

Weights can be used to balance the distribution of partitions on drives
across the cluster. This can be useful, for example, when different sized
drives are used in a cluster.

The ring is used by the Proxy server and several background processes
(like replication).

-------------
Object Server
-------------

The Object Server is a very simple blob storage server that can store,
retrieve and delete objects stored on local devices. Objects are stored
as binary files on the filesystem with metadata stored in the file's
extended attributes (xattrs). This requires that the underlying filesystem
choice for object servers support xattrs on files. Some filesystems,
like ext3, have xattrs turned off by default.

Each object is stored using a path derived from the object name's hash and
the operation's timestamp. Last write always wins, and ensures that the
latest object version will be served. A deletion is also treated as a
version of the file (a 0 byte file ending with ".ts", which stands for
tombstone). This ensures that deleted files are replicated correctly and
older versions don't magically reappear due to failure scenarios.

----------------
Container Server
----------------

The Container Server's primary job is to handle listings of objects. It
doesn't know where those object's are, just what objects are in a specific
container. The listings are stored as sqlite database files, and replicated
across the cluster similar to how objects are. Statistics are also tracked
that include the total number of objects, and total storage usage for that
container.

--------------
Account Server
--------------

The Account Server is very similar to the Container Server, excepting that
it is responsible for listings of containers rather than objects.

-----------
Replication
-----------

Replication is designed to keep the system in a consistent state in the face
of temporary error conditions like network outages or drive failures.

The replication processes compare local data with each remote copy to ensure
they all contain the latest version. Object replication uses a hash list to
quickly compare subsections of each partition, and container and account
replication use a combination of hashes and shared high water marks.

Replication updates are push based. For object replication, updating is
just a matter of rsyncing files to the peer. Account and container
replication push missing records over HTTP or rsync whole database files.

The replicator also ensures that data is removed from the system. When an
item (object, container, or account) is deleted, a tombstone is set as the
latest version of the item. The replicator will see the tombstone and ensure
that the item is removed from the entire system.

--------
Updaters
--------

There are times when container or account data can not be immediately
updated. This usually occurs during failure scenarios or periods of high
load. If an update fails, the update is queued locally on the filesystem,
and the updater will process the failed updates. This is where an eventual
consistency window will most likely come in to play. For example, suppose a
container server is under load and a new object is put in to the system. The
object will be immediately available for reads as soon as the proxy server
responds to the client with success. However, the container server did not
update the object listing, and so the update would be queued for a later
update. Container listings, therefore, may not immediately contain the object.

In practice, the consistency window is only as large as the frequency at
which the updater runs and may not even be noticed as the proxy server will
route listing requests to the first container server which responds. The
server under load may not be the one that serves subsequent listing
requests -- one of the other two replicas may handle the listing.

--------
Auditors
--------

Auditors crawl the local server checking the integrity of the objects,
containers, and accounts. If corruption is found (in the case of bit rot,
for example), the file is quarantined, and replication will replace the bad
file from another replica. If other errors are found they are logged (for
example, an object's listing can't be found on any container server it
should be).

swift-1.13.1/doc/source/development_middleware.rst0000664000175400017540000002240412323703611023405 0ustar  jenkinsjenkins00000000000000=======================
Middleware and Metadata
=======================

----------------
Using Middleware
----------------

`Python WSGI Middleware`_ (or just "middleware") can be used to "wrap"
the request and response of a Python WSGI application (i.e. a webapp,
or REST/HTTP API), like Swift's WSGI servers (proxy-server,
account-server, container-server, object-server).  Swift uses middleware
to add (sometimes optional) behaviors to the Swift WSGI servers.

.. _Python WSGI Middleware: http://www.python.org/dev/peps/pep-0333/#middleware-components-that-play-both-sides

Middleware can be added to the Swift WSGI servers by modifying their
`paste`_ configuration file.  The majority of Swift middleware is applied
to the :ref:`proxy-server`.

.. _paste: http://pythonpaste.org/

Given the following basic configuration::

    [DEFAULT]
    log_level = DEBUG
    user = 

    [pipeline:main]
    pipeline = proxy-server

    [app:proxy-server]
    use = egg:swift#proxy

You could add the :ref:`healthcheck` middleware by adding a section for
that filter and adding it to the pipeline::

    [DEFAULT]
    log_level = DEBUG
    user = 

    [pipeline:main]
    pipeline = healthcheck proxy-server

    [filter:healthcheck]
    use = egg:swift#healthcheck

    [app:proxy-server]
    use = egg:swift#proxy


Some middleware is required and will be inserted into your pipeline
automatically by core swift code (e.g. the proxy-server will insert
:ref:`catch_errors` and :ref:`gatekeeper` at the start of the pipeline if they
are not already present).  You can see which features are available on a given
Swift endpoint (including middleware) using the :ref:`discoverability`
interface.


----------------------------
Creating Your Own Middleware
----------------------------

The best way to see how to write middleware is to look at examples.

Many optional features in Swift are implemented as
:ref:`common_middleware` and provided in ``swift.common.middleware``, but
Swift middleware may be packaged and distributed as a separate project.
Some examples are listed on the :ref:`associated_projects` page.

A contrived middleware example that modifies request behavior by
inspecting custom HTTP headers (e.g. X-Webhook) and uses :ref:`sysmeta`
to persist data to backend storage as well as common patterns like a
:func:`.get_container_info` cache/query and :func:`.wsgify` decorator is
presented below::

    from swift.common.http import is_success
    from swift.common.swob import wsgify
    from swift.common.utils import split_path, get_logger
    from swift.common.request_helper import get_sys_meta_prefix
    from swift.proxy.controllers.base import get_container_info

    from eventlet import Timeout
    from eventlet.green import urllib2

    # x-container-sysmeta-webhook
    SYSMETA_WEBHOOK = get_sys_meta_prefix('container') + 'webhook'


    class WebhookMiddleware(object):

        def __init__(self, app, conf):
            self.app = app
            self.logger = get_logger(conf, log_route='webhook')

        @wsgify
        def __call__(self, req):
            obj = None
            try:
                (version, account, container, obj) = \
                    split_path(req.path_info, 4, 4, True)
            except ValueError:
                # not an object request
                pass
            if 'x-webhook' in req.headers:
                # translate user's request header to sysmeta
                req.headers[SYSMETA_WEBHOOK] = \
                    req.headers['x-webhook']
            if 'x-remove-webhook' in req.headers:
                # empty value will tombstone sysmeta
                req.headers[SYSMETA_WEBHOOK] = ''
            # account and object storage will ignore x-container-sysmeta-*
            resp = req.get_response(self.app)
            if obj and is_success(resp.status_int) and req.method == 'PUT':
                container_info = get_container_info(req.environ, self.app)
                # container_info may have our new sysmeta key
                webhook = container_info['sysmeta'].get('webhook')
                if webhook:
                    # create a POST request with obj name as body
                    webhook_req = urllib2.Request(webhook, data=obj)
                    with Timeout(20):
                        try:
                            urllib2.urlopen(webhook_req).read()
                        except (Exception, Timeout):
                            self.logger.exception(
                                'failed POST to webhook %s' % webhook)
                        else:
                            self.logger.info(
                                'successfully called webhook %s' % webhook)
            if 'x-container-sysmeta-webhook' in resp.headers:
                # translate sysmeta from the backend resp to
                # user-visible client resp header
                resp.headers['x-webhook'] = resp.headers[SYSMETA_WEBHOOK]
            return resp


    def webhook_factory(global_conf, **local_conf):
        conf = global_conf.copy()
        conf.update(local_conf)
        def webhook_filter(app, conf):
            return WebhookMiddleware(app)
        return webhook_filter

In practice this middleware will call the url stored on the container as
X-Webhook on all successful object uploads.

If this example was at ``/swift/common/middleware/webhook.py`` -
you could add it to your proxy by creating a new filter section and
adding it to the pipeline::

    [DEFAULT]
    log_level = DEBUG
    user = 

    [pipeline:main]
    pipeline = healthcheck webhook proxy-server

    [filter:webhook]
    paste.filter_factory = swift.common.middleware.webhook:webhook_factory

    [filter:healthcheck]
    use = egg:swift#healthcheck

    [app:proxy-server]
    use = egg:swift#proxy

Most python packages expose middleware as entrypoints.  See `PasteDeploy`_
documentation for more information about the syntax of the ``use`` option.
All middleware included with Swift is installed to support the ``egg:swift``
syntax.

.. _PasteDeploy: http://pythonpaste.org/deploy/#egg-uris

Middleware may advertize its availability and capabilities via Swift's
:ref:`discoverability` support by using
:func:`.register_swift_info`::

    from swift.common.utils import register_swift_info
    def webhook_factory(global_conf, **local_conf):
        register_swift_info('webhook')
        def webhook_filter(app):
            return WebhookMiddleware(app)
        return webhook_filter


--------------
Swift Metadata
--------------

Generally speaking metadata is information about a resource that is
associated with the resource but is not the data contained in the
resource itself - which is set and retrieved via HTTP headers. (e.g. the
"Content-Type" of a Swift object that is returned in HTTP response
headers)

All user resources in Swift (i.e. account, container, objects) can have
user metadata associated with them.  Middleware may also persist custom
metadata to accounts and containers safely using System Metadata.  Some
core swift features which predate sysmeta have added exceptions for
custom non-user metadata headers (e.g.  :ref:`acls`,
:ref:`large-objects`)

^^^^^^^^^^^^^
User Metadata
^^^^^^^^^^^^^

User metadata takes the form of ``X--Meta-: ``, where
```` depends on the resources type (i.e. Account, Container, Object)
and ```` and ```` are set by the client.

User metadata should generally be reserved for use by the client or
client applications.  An perfect example use-case for user metadata is
`python-swiftclient`_'s ``X-Object-Meta-Mtime`` which it stores on
object it uploads to implement its ``--changed`` option which will only
upload files that have changed since the last upload.

.. _python-swiftclient: https://github.com/openstack/python-swiftclient

New middleware should avoid storing metadata within the User Metadata
namespace to avoid potential conflict with existing user metadata when
introducing new metadata keys.  An example of legacy middleware that
borrows the user metadata namespace is :ref:`tempurl`.  An example of
middleware which uses custom non-user metadata to avoid the user
metadata namespace is :ref:`slo-doc`.

.. _sysmeta:

^^^^^^^^^^^^^^^
System Metadata
^^^^^^^^^^^^^^^

System metadata takes the form of ``X--Sysmeta-: ``,
where ```` depends on the resources type (i.e. Account, Container,
Object) and ```` and ```` are set by trusted code running in a
Swift WSGI Server.

All headers on client requests in the form of ``X--Sysmeta-``
will be dropped from the request before being processed by any
middleware.  All headers on responses from back-end systems in the form
of ``X--Sysmeta-`` will be removed after all middleware has
processed the response but before the response is sent to the client.
See :ref:`gatekeeper` middleware for more information.

System metadata provides a means to store potentially private custom
metadata with associated Swift resources in a safe and secure fashion
without actually having to plumb custom metadata through the core swift
servers.  The incoming filtering ensures that the namespace can not be
modified directly by client requests, and the outgoing filter ensures
that removing middleware that uses a specific system metadata key
renders it benign.  New middleware should take advantage of system
metadata.
swift-1.13.1/doc/source/_static/0000775000175400017540000000000012323703665017571 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/source/_static/basic.css0000664000175400017540000001462512323703611021363 0ustar  jenkinsjenkins00000000000000/**
 * Sphinx stylesheet -- basic theme
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */

/* -- main layout ----------------------------------------------------------- */

div.clearer {
    clear: both;
}

/* -- relbar ---------------------------------------------------------------- */

div.related {
    width: 100%;
    font-size: 90%;
}

div.related h3 {
    display: none;
}

div.related ul {
    margin: 0;
    padding: 0 0 0 10px;
    list-style: none;
}

div.related li {
    display: inline;
}

div.related li.right {
    float: right;
    margin-right: 5px;
}

/* -- sidebar --------------------------------------------------------------- */

div.sphinxsidebarwrapper {
    padding: 10px 5px 0 10px;
}

div.sphinxsidebar {
    float: left;
    width: 230px;
    margin-left: -100%;
    font-size: 90%;
}

div.sphinxsidebar ul {
    list-style: none;
}

div.sphinxsidebar ul ul,
div.sphinxsidebar ul.want-points {
    margin-left: 20px;
    list-style: square;
}

div.sphinxsidebar ul ul {
    margin-top: 0;
    margin-bottom: 0;
}

div.sphinxsidebar form {
    margin-top: 10px;
}

div.sphinxsidebar input {
    border: 1px solid #98dbcc;
    font-family: sans-serif;
    font-size: 1em;
}

img {
    border: 0;
}

/* -- search page ----------------------------------------------------------- */

ul.search {
    margin: 10px 0 0 20px;
    padding: 0;
}

ul.search li {
    padding: 5px 0 5px 20px;
    background-image: url(file.png);
    background-repeat: no-repeat;
    background-position: 0 7px;
}

ul.search li a {
    font-weight: bold;
}

ul.search li div.context {
    color: #888;
    margin: 2px 0 0 30px;
    text-align: left;
}

ul.keywordmatches li.goodmatch a {
    font-weight: bold;
}

/* -- index page ------------------------------------------------------------ */

table.contentstable {
    width: 90%;
}

table.contentstable p.biglink {
    line-height: 150%;
}

a.biglink {
    font-size: 1.3em;
}

span.linkdescr {
    font-style: italic;
    padding-top: 5px;
    font-size: 90%;
}

/* -- general index --------------------------------------------------------- */

table.indextable td {
    text-align: left;
    vertical-align: top;
}

table.indextable dl, table.indextable dd {
    margin-top: 0;
    margin-bottom: 0;
}

table.indextable tr.pcap {
    height: 10px;
}

table.indextable tr.cap {
    margin-top: 10px;
    background-color: #f2f2f2;
}

img.toggler {
    margin-right: 3px;
    margin-top: 3px;
    cursor: pointer;
}

/* -- general body styles --------------------------------------------------- */

a.headerlink {
    visibility: hidden;
}

h1:hover > a.headerlink,
h2:hover > a.headerlink,
h3:hover > a.headerlink,
h4:hover > a.headerlink,
h5:hover > a.headerlink,
h6:hover > a.headerlink,
dt:hover > a.headerlink {
    visibility: visible;
}

div.body p.caption {
    text-align: inherit;
}

div.body td {
    text-align: left;
}

.field-list ul {
    padding-left: 1em;
}

.first {
}

p.rubric {
    margin-top: 30px;
    font-weight: bold;
}

/* -- sidebars -------------------------------------------------------------- */

div.sidebar {
    margin: 0 0 0.5em 1em;
    border: 1px solid #ddb;
    padding: 7px 7px 0 7px;
    background-color: #ffe;
    width: 40%;
    float: right;
}

p.sidebar-title {
    font-weight: bold;
}

/* -- topics ---------------------------------------------------------------- */

div.topic {
    border: 1px solid #ccc;
    padding: 7px 7px 0 7px;
    margin: 10px 0 10px 0;
}

p.topic-title {
    font-size: 1.1em;
    font-weight: bold;
    margin-top: 10px;
}

/* -- admonitions ----------------------------------------------------------- */

div.admonition {
    margin-top: 10px;
    margin-bottom: 10px;
    padding: 7px;
}

div.admonition dt {
    font-weight: bold;
}

div.admonition dl {
    margin-bottom: 0;
}

p.admonition-title {
    margin: 0px 10px 5px 0px;
    font-weight: bold;
}

div.body p.centered {
    text-align: center;
    margin-top: 25px;
}

/* -- tables ---------------------------------------------------------------- */

table.docutils {
    border: 0;
    border-collapse: collapse;
}

table.docutils td, table.docutils th {
    padding: 1px 8px 1px 0;
    border-top: 0;
    border-left: 0;
    border-right: 0;
    border-bottom: 1px solid #aaa;
}

table.field-list td, table.field-list th {
    border: 0 !important;
}

table.footnote td, table.footnote th {
    border: 0 !important;
}

th {
    text-align: left;
    padding-right: 5px;
}

/* -- other body styles ----------------------------------------------------- */

dl {
    margin-bottom: 15px;
}

dd p {
    margin-top: 0px;
}

dd ul, dd table {
    margin-bottom: 10px;
}

dd {
    margin-top: 3px;
    margin-bottom: 10px;
    margin-left: 30px;
}

dt:target, .highlight {
    background-color: #fbe54e;
}

dl.glossary dt {
    font-weight: bold;
    font-size: 1.1em;
}

.field-list ul {
    margin: 0;
    padding-left: 1em;
}

.field-list p {
    margin: 0;
}

.refcount {
    color: #060;
}

.optional {
    font-size: 1.3em;
}

.versionmodified {
    font-style: italic;
}

.system-message {
    background-color: #fda;
    padding: 5px;
    border: 3px solid red;
}

.footnote:target  {
    background-color: #ffa
}

.line-block {
    display: block;
    margin-top: 1em;
    margin-bottom: 1em;
}

.line-block .line-block {
    margin-top: 0;
    margin-bottom: 0;
    margin-left: 1.5em;
}

/* -- code displays --------------------------------------------------------- */

pre {
    overflow: auto;
}

td.linenos pre {
    padding: 5px 0px;
    border: 0;
    background-color: transparent;
    color: #aaa;
}

table.highlighttable {
    margin-left: 0.5em;
}

table.highlighttable td {
    padding: 0 0.5em 0 0.5em;
}

tt.descname {
    background-color: transparent;
    font-weight: bold;
    font-size: 1.2em;
}

tt.descclassname {
    background-color: transparent;
}

tt.xref, a tt {
    background-color: transparent;
    font-weight: bold;
}

h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
    background-color: transparent;
}

/* -- math display ---------------------------------------------------------- */

img.math {
    vertical-align: middle;
}

div.body div.math p {
    text-align: center;
}

span.eqno {
    float: right;
}

/* -- printout stylesheet --------------------------------------------------- */

@media print {
    div.document,
    div.documentwrapper,
    div.bodywrapper {
        margin: 0 !important;
        width: 100%;
    }

    div.sphinxsidebar,
    div.related,
    div.footer,
    #top-link {
        display: none;
    }
}
swift-1.13.1/doc/source/_static/default.css0000664000175400017540000000707712323703611021731 0ustar  jenkinsjenkins00000000000000/**
 * Sphinx stylesheet -- default theme
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */

@import url("basic.css");

/* -- page layout ----------------------------------------------------------- */

body {
    font-family: sans-serif;
    font-size: 100%;
    background-color: #11303d;
    color: #000;
    margin: 0;
    padding: 0;
}

div.document {
    background-color: #1c4e63;
}

div.documentwrapper {
    float: left;
    width: 100%;
}

div.bodywrapper {
    margin: 0 0 0 230px;
}

div.body {
    background-color: #ffffff;
    color: #000000;
    padding: 0 20px 30px 20px;
}

div.footer {
    color: #ffffff;
    width: 100%;
    padding: 9px 0 9px 0;
    text-align: center;
    font-size: 75%;
}

div.footer a {
    color: #ffffff;
    text-decoration: underline;
}

div.related {
    background-color: #133f52;
    line-height: 30px;
    color: #ffffff;
}

div.related a {
    color: #ffffff;
}

div.sphinxsidebar {
}

div.sphinxsidebar h3 {
    font-family: 'Trebuchet MS', sans-serif;
    color: #ffffff;
    font-size: 1.4em;
    font-weight: normal;
    margin: 0;
    padding: 0;
}

div.sphinxsidebar h3 a {
    color: #ffffff;
}

div.sphinxsidebar h4 {
    font-family: 'Trebuchet MS', sans-serif;
    color: #ffffff;
    font-size: 1.3em;
    font-weight: normal;
    margin: 5px 0 0 0;
    padding: 0;
}

div.sphinxsidebar p {
    color: #ffffff;
}

div.sphinxsidebar p.topless {
    margin: 5px 10px 10px 10px;
}

div.sphinxsidebar ul {
    margin: 10px;
    padding: 0;
    color: #ffffff;
}

div.sphinxsidebar a {
    color: #98dbcc;
}

div.sphinxsidebar input {
    border: 1px solid #98dbcc;
    font-family: sans-serif;
    font-size: 1em;
}

/* -- body styles ----------------------------------------------------------- */

a {
    color: #355f7c;
    text-decoration: none;
}

a:hover {
    text-decoration: underline;
}

div.body p, div.body dd, div.body li {
    text-align: left;
    line-height: 130%;
}

div.body h1,
div.body h2,
div.body h3,
div.body h4,
div.body h5,
div.body h6 {
    font-family: 'Trebuchet MS', sans-serif;
    background-color: #f2f2f2;
    font-weight: normal;
    color: #20435c;
    border-bottom: 1px solid #ccc;
    margin: 20px -20px 10px -20px;
    padding: 3px 0 3px 10px;
}

div.body h1 { margin-top: 0; font-size: 200%; }
div.body h2 { font-size: 160%; }
div.body h3 { font-size: 140%; }
div.body h4 { font-size: 120%; }
div.body h5 { font-size: 110%; }
div.body h6 { font-size: 100%; }

a.headerlink {
    color: #c60f0f;
    font-size: 0.8em;
    padding: 0 4px 0 4px;
    text-decoration: none;
}

a.headerlink:hover {
    background-color: #c60f0f;
    color: white;
}

div.body p, div.body dd, div.body li {
    text-align: left;
    line-height: 130%;
}

div.admonition p.admonition-title + p {
    display: inline;
}

div.admonition p {
    margin-bottom: 5px;
}

div.admonition pre {
    margin-bottom: 5px;
}

div.admonition ul, div.admonition ol {
    margin-bottom: 5px;
}

div.note {
    background-color: #eee;
    border: 1px solid #ccc;
}

div.seealso {
    background-color: #ffc;
    border: 1px solid #ff6;
}

div.topic {
    background-color: #eee;
}

div.warning {
    background-color: #ffe4e4;
    border: 1px solid #f66;
}

p.admonition-title {
    display: inline;
}

p.admonition-title:after {
    content: ":";
}

pre {
    padding: 5px;
    background-color: #eeffcc;
    color: #333333;
    line-height: 120%;
    border: 1px solid #ac9;
    border-left: none;
    border-right: none;
}

tt {
    background-color: #ecf0f3;
    padding: 0 1px 0 1px;
    font-size: 0.95em;
}

.warning tt {
    background: #efc2c2;
}

.note tt {
    background: #d6d6d6;
}
swift-1.13.1/doc/source/_static/tweaks.css0000664000175400017540000001136412323703611021575 0ustar  jenkinsjenkins00000000000000ul.todo_list {
    list-style-type: none;
    margin: 0;
    padding: 0;
}

ul.todo_list li {
    display: block;
    margin: 0;
    padding: 7px 0;
    border-top: 1px solid #eee;
}

ul.todo_list li p {
    display: inline;
}

ul.todo_list li p.link {
    font-weight: bold;
}

ul.todo_list li p.details {
    font-style: italic;
}

ul.todo_list li {
}

div.admonition {
    border: 1px solid #8F1000;
}

div.admonition p.admonition-title {
    background-color: #8F1000;
    border-bottom: 1px solid #8E8E8E;
}

a {
	color: #CF2F19;
}

div.related ul li a {
	color: #CF2F19;
}

div.sphinxsidebar h4 {
	background-color:#8E8E8E;
	border:1px solid #255E6E;
	color:white;
	font-size:1em;
	margin:1em 0 0.5em;
	padding:0.1em 0 0.1em 0.5em;
}

em {
    font-style: normal;
}

table.docutils {
	font-size: 11px;
}

a tt {
	color:#CF2F19;
}

/* ------------------------------------------
PURE CSS SPEECH BUBBLES
by Nicolas Gallagher
- http://nicolasgallagher.com/pure-css-speech-bubbles/

http://nicolasgallagher.com
http://twitter.com/necolas

Created: 02 March 2010
Version: 1.1 (21 October 2010)

Dual licensed under MIT and GNU GPLv2 © Nicolas Gallagher 
------------------------------------------ */
/* THE SPEECH BUBBLE
------------------------------------------------------------------------------------------------------------------------------- */

/* THE SPEECH BUBBLE
------------------------------------------------------------------------------------------------------------------------------- */

.triangle-border {
	position:relative;
	padding:15px;
	margin:1em 0 3em;
	border:5px solid #BC1518;
	color:#333;
	background:#fff;
	
	/* css3 */
	-moz-border-radius:10px;
	-webkit-border-radius:10px;
	border-radius:10px;
}

/* Variant : for left positioned triangle
------------------------------------------ */

.triangle-border.left {
	margin-left:30px;
}

/* Variant : for right positioned triangle
------------------------------------------ */

.triangle-border.right {
	margin-right:30px;
}

/* THE TRIANGLE
------------------------------------------------------------------------------------------------------------------------------- */

.triangle-border:before {
	content:"";
	display:block; /* reduce the damage in FF3.0 */
	position:absolute;
	bottom:-40px; /* value = - border-top-width - border-bottom-width */
	left:40px; /* controls horizontal position */
	width:0;
	height:0;
	border:20px solid transparent;
	border-top-color:#BC1518;
}

/* creates the smaller  triangle */
.triangle-border:after {
	content:"";
	display:block; /* reduce the damage in FF3.0 */
	position:absolute;
	bottom:-26px; /* value = - border-top-width - border-bottom-width */
	left:47px; /* value = (:before left) + (:before border-left) - (:after border-left) */
	width:0;
	height:0;
	border:13px solid transparent;
	border-top-color:#fff;
}

/* Variant : top
------------------------------------------ */

/* creates the larger triangle */
.triangle-border.top:before {
	top:-40px; /* value = - border-top-width - border-bottom-width */
	right:40px; /* controls horizontal position */
	bottom:auto;
	left:auto;
	border:20px solid transparent;
	border-bottom-color:#BC1518;
}

/* creates the smaller  triangle */
.triangle-border.top:after {
	top:-26px; /* value = - border-top-width - border-bottom-width */
	right:47px; /* value = (:before right) + (:before border-right) - (:after border-right) */
	bottom:auto;
	left:auto;
	border:13px solid transparent;
	border-bottom-color:#fff;
}

/* Variant : left
------------------------------------------ */

/* creates the larger triangle */
.triangle-border.left:before {
	top:10px; /* controls vertical position */
	left:-30px; /* value = - border-left-width - border-right-width */
	bottom:auto;
	border-width:15px 30px 15px 0;
	border-style:solid;
	border-color:transparent #BC1518;
}

/* creates the smaller  triangle */
.triangle-border.left:after {
	top:16px; /* value = (:before top) + (:before border-top) - (:after border-top) */
	left:-21px; /* value = - border-left-width - border-right-width */
	bottom:auto;
	border-width:9px 21px 9px 0;
	border-style:solid;
	border-color:transparent #fff;
}

/* Variant : right
------------------------------------------ */

/* creates the larger triangle */
.triangle-border.right:before {
	top:10px; /* controls vertical position */
	right:-30px; /* value = - border-left-width - border-right-width */
	bottom:auto;
    left:auto;
	border-width:15px 0 15px 30px;
	border-style:solid;
	border-color:transparent #BC1518;
}

/* creates the smaller  triangle */
.triangle-border.right:after {
	top:16px; /* value = (:before top) + (:before border-top) - (:after border-top) */
	right:-21px; /* value = - border-left-width - border-right-width */
	bottom:auto;
    left:auto;
	border-width:9px 0 9px 21px;
	border-style:solid;
	border-color:transparent #fff;
}

swift-1.13.1/doc/source/overview_replication.rst0000664000175400017540000001446112323703611023131 0ustar  jenkinsjenkins00000000000000===========
Replication
===========

Because each replica in swift functions independently, and clients generally
require only a simple majority of nodes responding to consider an operation
successful, transient failures like network partitions can quickly cause
replicas to diverge. These differences are eventually reconciled by
asynchronous, peer-to-peer replicator processes. The replicator processes
traverse their local filesystems, concurrently performing operations in a
manner that balances load across physical disks.

Replication uses a push model, with records and files generally only being
copied from local to remote replicas. This is important because data on the
node may not belong there (as in the case of handoffs and ring changes), and a
replicator can't know what data exists elsewhere in the cluster that it should
pull in. It's the duty of any node that contains data to ensure that data gets
to where it belongs. Replica placement is handled by the ring.

Every deleted record or file in the system is marked by a tombstone, so that
deletions can be replicated alongside creations. The replication process cleans
up tombstones after a time period known as the consistency window.
The consistency window encompasses replication duration and how long transient
failure can remove a node from the cluster. Tombstone cleanup must
be tied to replication to reach replica convergence.

If a replicator detects that a remote drive has failed, the replicator uses
the get_more_nodes interface for the ring to choose an alternate node with
which to synchronize. The replicator can maintain desired levels of replication
in the face of disk failures, though some replicas may not be in an immediately
usable location. Note that the replicator doesn't maintain desired levels of
replication when other failures, such as entire node failures, occur because
most failure are transient.

Replication is an area of active development, and likely rife with potential
improvements to speed and correctness.

There are two major classes of replicator - the db replicator, which
replicates accounts and containers, and the object replicator, which
replicates object data.

--------------
DB Replication
--------------

The first step performed by db replication is a low-cost hash comparison to
determine whether two replicas already match. Under normal operation,
this check is able to verify that most databases in the system are already
synchronized very quickly. If the hashes differ, the replicator brings the
databases in sync by sharing records added since the last sync point.

This sync point is a high water mark noting the last record at which two
databases were known to be in sync, and is stored in each database as a tuple
of the remote database id and record id. Database ids are unique amongst all
replicas of the database, and record ids are monotonically increasing
integers. After all new records have been pushed to the remote database, the
entire sync table of the local database is pushed, so the remote database
can guarantee that it is in sync with everything with which the local database
has previously synchronized.

If a replica is found to be missing entirely, the whole local database file is
transmitted to the peer using rsync(1) and vested with a new unique id.

In practice, DB replication can process hundreds of databases per concurrency
setting per second (up to the number of available CPUs or disks) and is bound
by the number of DB transactions that must be performed.


------------------
Object Replication
------------------

The initial implementation of object replication simply performed an rsync to
push data from a local partition to all remote servers it was expected to
exist on. While this performed adequately at small scale, replication times
skyrocketed once directory structures could no longer be held in RAM. We now
use a modification of this scheme in which a hash of the contents for each
suffix directory is saved to a per-partition hashes file. The hash for a
suffix directory is invalidated when the contents of that suffix directory are
modified.

The object replication process reads in these hash files, calculating any
invalidated hashes. It then transmits the hashes to each remote server that
should hold the partition, and only suffix directories with differing hashes
on the remote server are rsynced. After pushing files to the remote server,
the replication process notifies it to recalculate hashes for the rsynced
suffix directories.

Performance of object replication is generally bound by the number of uncached
directories it has to traverse, usually as a result of invalidated suffix
directory hashes. Using write volume and partition counts from our running
systems, it was designed so that around 2% of the hash space on a normal node
will be invalidated per day, which has experimentally given us acceptable
replication speeds.

Work continues with a new ssync method where rsync is not used at all and
instead all-Swift code is used to transfer the objects. At first, this ssync
will just strive to emulate the rsync behavior. Once deemed stable it will open
the way for future improvements in replication since we'll be able to easily
add code in the replication path instead of trying to alter the rsync code
base and distributing such modifications.

One of the first improvements planned is an "index.db" that will replace the
hashes.pkl. This will allow quicker updates to that data as well as more
streamlined queries. Quite likely we'll implement a better scheme than the
current one hashes.pkl uses (hash-trees, that sort of thing).

Another improvement planned all along the way is separating the local disk
structure from the protocol path structure. This separation will allow ring
resizing at some point, or at least ring-doubling.

FOR NOW, IT IS NOT RECOMMENDED TO USE SSYNC ON PRODUCTION CLUSTERS. Some of us
will be in a limited fashion to look for any subtle issues, tuning, etc. but
generally ssync is an experimental feature. In its current implementation it is
probably going to be a bit slower than RSync, but if all goes according to plan
it will end up much faster.


-----------------------------
Dedicated replication network
-----------------------------

Swift has support for using dedicated network for replication traffic.
For more information see :ref:`Overview of dedicated replication network
`.
swift-1.13.1/doc/source/overview_expiring_objects.rst0000664000175400017540000000640612323703611024156 0ustar  jenkinsjenkins00000000000000=======================
Expiring Object Support
=======================

The ``swift-object-expirer`` offers scheduled deletion of objects. The Swift client would use the ``X-Delete-At`` or ``X-Delete-After`` headers during an object ``PUT`` or ``POST`` and the cluster would automatically quit serving that object at the specified time and would shortly thereafter remove the object from the system.

The ``X-Delete-At`` header takes a Unix Epoch timestamp, in integer form; for example: ``1317070737`` represents ``Mon Sep 26 20:58:57 2011 UTC``.

The ``X-Delete-After`` header takes a integer number of seconds. The proxy server that receives the request will convert this header into an ``X-Delete-At`` header using its current time plus the value given.

As expiring objects are added to the system, the object servers will record the expirations in a hidden ``.expiring_objects`` account for the ``swift-object-expirer`` to handle later.

Usually, just one instance of the ``swift-object-expirer`` daemon needs to run for a cluster. This isn't exactly automatic failover high availability, but if this daemon doesn't run for a few hours it should not be any real issue. The expired-but-not-yet-deleted objects will still ``404 Not Found`` if someone tries to ``GET`` or ``HEAD`` them and they'll just be deleted a bit later when the daemon is restarted.

By default, the ``swift-object-expirer`` daemon will run with a concurrency of 1.  Increase this value to get more concurrency.  A concurrency of 1 may not be enough to delete expiring objects in a timely fashion for a particular swift cluster.

It is possible to run multiple daemons to do different parts of the work if a single process with a concurrency of more than 1 is not enough (see the sample config file for details).

To run the ``swift-object-expirer`` as multiple processes, set ``processes`` to the number of processes (either in the config file or on the command line).  Then run one process for each part.  Use ``process`` to specify the part of the work to be done by a process using the command line or the config.  So, for example, if you'd like to run three processes, set ``processes`` to 3 and run three processes with ``process`` set to 0, 1, and 2 for the three processes.  If multiple processes are used, it's necessary to run one for each part of the work or that part of the work will not be done.

The daemon uses the ``/etc/swift/object-expirer.conf`` by default, and here is a quick sample conf file::

    [DEFAULT]
    # swift_dir = /etc/swift
    # user = swift
    # You can specify default log routing here if you want:
    # log_name = swift
    # log_facility = LOG_LOCAL0
    # log_level = INFO

    [object-expirer]
    interval = 300

    [pipeline:main]
    pipeline = catch_errors cache proxy-server

    [app:proxy-server]
    use = egg:swift#proxy
    # See proxy-server.conf-sample for options

    [filter:cache]
    use = egg:swift#memcache
    # See proxy-server.conf-sample for options

    [filter:catch_errors]
    use = egg:swift#catch_errors
    # See proxy-server.conf-sample for options

The daemon needs to run on a machine with access to all the backend servers in the cluster, but does not need proxy server or public access. The daemon will use its own internal proxy code instance to access the backend servers.
swift-1.13.1/doc/saio/0000775000175400017540000000000012323703665015576 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/rsyslog.d/0000775000175400017540000000000012323703665017522 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/rsyslog.d/10-swift.conf0000664000175400017540000000213412323703611021732 0ustar  jenkinsjenkins00000000000000# Uncomment the following to have a log containing all logs together
#local1,local2,local3,local4,local5.*   /var/log/swift/all.log

# Uncomment the following to have hourly proxy logs for stats processing
#$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
#local1.*;local1.!notice ?HourlyProxyLog

local1.*;local1.!notice /var/log/swift/proxy.log
local1.notice           /var/log/swift/proxy.error
local1.*                ~

local2.*;local2.!notice /var/log/swift/storage1.log
local2.notice           /var/log/swift/storage1.error
local2.*                ~

local3.*;local3.!notice /var/log/swift/storage2.log
local3.notice           /var/log/swift/storage2.error
local3.*                ~

local4.*;local4.!notice /var/log/swift/storage3.log
local4.notice           /var/log/swift/storage3.error
local4.*                ~

local5.*;local5.!notice /var/log/swift/storage4.log
local5.notice           /var/log/swift/storage4.error
local5.*                ~

local6.*;local6.!notice /var/log/swift/expirer.log
local6.notice           /var/log/swift/expirer.error
local6.*                ~
swift-1.13.1/doc/saio/swift/0000775000175400017540000000000012323703665016732 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/swift/proxy-server.conf0000664000175400017540000000251612323703611022261 0ustar  jenkinsjenkins00000000000000[DEFAULT]
bind_port = 8080
workers = 1
user = 
log_facility = LOG_LOCAL1
eventlet_debug = true

[pipeline:main]
# Yes, proxy-logging appears twice. This is so that
# middleware-originated requests get logged too.
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl slo dlo ratelimit crossdomain tempauth staticweb container-quotas account-quotas proxy-logging proxy-server

[filter:catch_errors]
use = egg:swift#catch_errors

[filter:healthcheck]
use = egg:swift#healthcheck

[filter:proxy-logging]
use = egg:swift#proxy_logging

[filter:bulk]
use = egg:swift#bulk

[filter:ratelimit]
use = egg:swift#ratelimit

[filter:crossdomain]
use = egg:swift#crossdomain

[filter:dlo]
use = egg:swift#dlo

[filter:slo]
use = egg:swift#slo

[filter:tempurl]
use = egg:swift#tempurl

[filter:tempauth]
use = egg:swift#tempauth
user_admin_admin = admin .admin .reseller_admin
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3

[filter:staticweb]
use = egg:swift#staticweb

[filter:account-quotas]
use = egg:swift#account_quotas

[filter:container-quotas]
use = egg:swift#container_quotas

[filter:cache]
use = egg:swift#memcache

[filter:gatekeeper]
use = egg:swift#gatekeeper

[app:proxy-server]
use = egg:swift#proxy
allow_account_management = true
account_autocreate = true
swift-1.13.1/doc/saio/swift/object-expirer.conf0000664000175400017540000000341512323703611022515 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# swift_dir = /etc/swift
user = 
# You can specify default log routing here if you want:
log_name = object-expirer
log_facility = LOG_LOCAL6
log_level = INFO
#log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =

[object-expirer]
interval = 300
# auto_create_account_prefix = .
# report_interval = 300
# concurrency is the level of concurrency o use to do the work, this value
# must be set to at least 1
# concurrency = 1
# processes is how many parts to divide the work into, one part per process
#   that will be doing the work
# processes set 0 means that a single process will be doing all the work
# processes can also be specified on the command line and will override the
#   config value
# processes = 0
# process is which of the parts a particular process will work on
# process can also be specified on the command line and will overide the config
#   value
# process is "zero based", if you want to use 3 processes, you should run
#  processes with process set to 0, 1, and 2
# process = 0

[pipeline:main]
pipeline = catch_errors cache proxy-server

[app:proxy-server]
use = egg:swift#proxy
# See proxy-server.conf-sample for options

[filter:cache]
use = egg:swift#memcache
# See proxy-server.conf-sample for options

[filter:catch_errors]
use = egg:swift#catch_errors
# See proxy-server.conf-sample for options
swift-1.13.1/doc/saio/swift/account-server/0000775000175400017540000000000012323703665021672 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/swift/account-server/3.conf0000664000175400017540000000065212323703611022675 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6032
workers = 1
user = 
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3
eventlet_debug = true

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account

[filter:recon]
use = egg:swift#recon

[account-replicator]
vm_test_mode = yes

[account-auditor]

[account-reaper]
swift-1.13.1/doc/saio/swift/account-server/2.conf0000664000175400017540000000065212323703611022674 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6022
workers = 1
user = 
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2
eventlet_debug = true

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account

[filter:recon]
use = egg:swift#recon

[account-replicator]
vm_test_mode = yes

[account-auditor]

[account-reaper]
swift-1.13.1/doc/saio/swift/account-server/4.conf0000664000175400017540000000065212323703611022676 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6042
workers = 1
user = 
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4
eventlet_debug = true

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account

[filter:recon]
use = egg:swift#recon

[account-replicator]
vm_test_mode = yes

[account-auditor]

[account-reaper]
swift-1.13.1/doc/saio/swift/account-server/1.conf0000664000175400017540000000065112323703611022672 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6012
workers = 1
user = 
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true

[pipeline:main]
pipeline = recon account-server

[app:account-server]
use = egg:swift#account

[filter:recon]
use = egg:swift#recon

[account-replicator]
vm_test_mode = yes

[account-auditor]

[account-reaper]
swift-1.13.1/doc/saio/swift/object-server/0000775000175400017540000000000012323703665021504 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/swift/object-server/3.conf0000664000175400017540000000064512323703611022511 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6030
workers = 1
user = 
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3
eventlet_debug = true

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object

[filter:recon]
use = egg:swift#recon

[object-replicator]
vm_test_mode = yes

[object-updater]

[object-auditor]
swift-1.13.1/doc/saio/swift/object-server/2.conf0000664000175400017540000000064512323703611022510 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6020
workers = 1
user = 
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2
eventlet_debug = true

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object

[filter:recon]
use = egg:swift#recon

[object-replicator]
vm_test_mode = yes

[object-updater]

[object-auditor]
swift-1.13.1/doc/saio/swift/object-server/4.conf0000664000175400017540000000064512323703611022512 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6040
workers = 1
user = 
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4
eventlet_debug = true

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object

[filter:recon]
use = egg:swift#recon

[object-replicator]
vm_test_mode = yes

[object-updater]

[object-auditor]
swift-1.13.1/doc/saio/swift/object-server/1.conf0000664000175400017540000000064412323703611022506 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6010
workers = 1
user = 
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true

[pipeline:main]
pipeline = recon object-server

[app:object-server]
use = egg:swift#object

[filter:recon]
use = egg:swift#recon

[object-replicator]
vm_test_mode = yes

[object-updater]

[object-auditor]
swift-1.13.1/doc/saio/swift/container-server/0000775000175400017540000000000012323703665022220 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/swift/container-server/3.conf0000664000175400017540000000073712323703611023227 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/3/node
mount_check = false
disable_fallocate = true
bind_port = 6031
workers = 1
user = 
log_facility = LOG_LOCAL4
recon_cache_path = /var/cache/swift3
eventlet_debug = true
allow_versions = true

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container

[filter:recon]
use = egg:swift#recon

[container-replicator]
vm_test_mode = yes

[container-updater]

[container-auditor]

[container-sync]
swift-1.13.1/doc/saio/swift/container-server/2.conf0000664000175400017540000000073712323703611023226 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/2/node
mount_check = false
disable_fallocate = true
bind_port = 6021
workers = 1
user = 
log_facility = LOG_LOCAL3
recon_cache_path = /var/cache/swift2
eventlet_debug = true
allow_versions = true

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container

[filter:recon]
use = egg:swift#recon

[container-replicator]
vm_test_mode = yes

[container-updater]

[container-auditor]

[container-sync]
swift-1.13.1/doc/saio/swift/container-server/4.conf0000664000175400017540000000073712323703611023230 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/4/node
mount_check = false
disable_fallocate = true
bind_port = 6041
workers = 1
user = 
log_facility = LOG_LOCAL5
recon_cache_path = /var/cache/swift4
eventlet_debug = true
allow_versions = true

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container

[filter:recon]
use = egg:swift#recon

[container-replicator]
vm_test_mode = yes

[container-updater]

[container-auditor]

[container-sync]
swift-1.13.1/doc/saio/swift/container-server/1.conf0000664000175400017540000000073612323703611023224 0ustar  jenkinsjenkins00000000000000[DEFAULT]
devices = /srv/1/node
mount_check = false
disable_fallocate = true
bind_port = 6011
workers = 1
user = 
log_facility = LOG_LOCAL2
recon_cache_path = /var/cache/swift
eventlet_debug = true
allow_versions = true

[pipeline:main]
pipeline = recon container-server

[app:container-server]
use = egg:swift#container

[filter:recon]
use = egg:swift#recon

[container-replicator]
vm_test_mode = yes

[container-updater]

[container-auditor]

[container-sync]
swift-1.13.1/doc/saio/swift/swift.conf0000664000175400017540000000021512323703611020722 0ustar  jenkinsjenkins00000000000000[swift-hash]
# random unique strings that can never change (DO NOT LOSE)
swift_hash_path_prefix = changeme
swift_hash_path_suffix = changeme
swift-1.13.1/doc/saio/rsyncd.conf0000664000175400017540000000272412323703611017743 0ustar  jenkinsjenkins00000000000000uid = 
gid = 
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid
address = 127.0.0.1

[account6012]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/account6012.lock

[account6022]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/account6022.lock

[account6032]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/account6032.lock

[account6042]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/account6042.lock

[container6011]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/container6011.lock

[container6021]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/container6021.lock

[container6031]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/container6031.lock

[container6041]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/container6041.lock

[object6010]
max connections = 25
path = /srv/1/node/
read only = false
lock file = /var/lock/object6010.lock

[object6020]
max connections = 25
path = /srv/2/node/
read only = false
lock file = /var/lock/object6020.lock

[object6030]
max connections = 25
path = /srv/3/node/
read only = false
lock file = /var/lock/object6030.lock

[object6040]
max connections = 25
path = /srv/4/node/
read only = false
lock file = /var/lock/object6040.lock
swift-1.13.1/doc/saio/bin/0000775000175400017540000000000012323703665016346 5ustar  jenkinsjenkins00000000000000swift-1.13.1/doc/saio/bin/resetswift0000775000175400017540000000134712323703611020467 0ustar  jenkinsjenkins00000000000000#!/bin/bash

swift-init all stop
# Remove the following line if you did not set up rsyslog for individual logging:
sudo find /var/log/swift -type f -exec rm -f {} \;
sudo umount /mnt/sdb1
# If you are using a loopback device substitute "/dev/sdb1" with "/srv/swift-disk"
sudo mkfs.xfs -f /dev/sdb1
sudo mount /mnt/sdb1
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
sudo chown ${USER}:${USER} /mnt/sdb1/*
mkdir -p /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4
sudo rm -f /var/log/debug /var/log/messages /var/log/rsyncd.log /var/log/syslog
find /var/cache/swift* -type f -name *.recon -exec rm -f {} \;
# On Fedora use "systemctl restart "
sudo service rsyslog restart
sudo service memcached restart
swift-1.13.1/doc/saio/bin/startrest0000775000175400017540000000004212323703611020312 0ustar  jenkinsjenkins00000000000000#!/bin/bash

swift-init rest startswift-1.13.1/doc/saio/bin/remakerings0000775000175400017540000000222312323703611020571 0ustar  jenkinsjenkins00000000000000#!/bin/bash

cd /etc/swift

rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz

swift-ring-builder object.builder create 10 3 1
swift-ring-builder object.builder add r1z1-127.0.0.1:6010/sdb1 1
swift-ring-builder object.builder add r1z2-127.0.0.1:6020/sdb2 1
swift-ring-builder object.builder add r1z3-127.0.0.1:6030/sdb3 1
swift-ring-builder object.builder add r1z4-127.0.0.1:6040/sdb4 1
swift-ring-builder object.builder rebalance
swift-ring-builder container.builder create 10 3 1
swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1
swift-ring-builder container.builder add r1z2-127.0.0.1:6021/sdb2 1
swift-ring-builder container.builder add r1z3-127.0.0.1:6031/sdb3 1
swift-ring-builder container.builder add r1z4-127.0.0.1:6041/sdb4 1
swift-ring-builder container.builder rebalance
swift-ring-builder account.builder create 10 3 1
swift-ring-builder account.builder add r1z1-127.0.0.1:6012/sdb1 1
swift-ring-builder account.builder add r1z2-127.0.0.1:6022/sdb2 1
swift-ring-builder account.builder add r1z3-127.0.0.1:6032/sdb3 1
swift-ring-builder account.builder add r1z4-127.0.0.1:6042/sdb4 1
swift-ring-builder account.builder rebalance
swift-1.13.1/doc/saio/bin/startmain0000775000175400017540000000004212323703611020261 0ustar  jenkinsjenkins00000000000000#!/bin/bash

swift-init main startswift-1.13.1/setup.cfg0000664000175400017540000000720412323703665015722 0ustar  jenkinsjenkins00000000000000[metadata]
name = swift
summary = OpenStack Object Storage
description-file = 
	README.md
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier = 
	Development Status :: 5 - Production/Stable
	Environment :: OpenStack
	Intended Audience :: Information Technology
	Intended Audience :: System Administrators
	License :: OSI Approved :: Apache Software License
	Operating System :: POSIX :: Linux
	Programming Language :: Python
	Programming Language :: Python :: 2
	Programming Language :: Python :: 2.6
	Programming Language :: Python :: 2.7

[pbr]
skip_authors = True
skip_changelog = True

[files]
packages = 
	swift
scripts = 
	bin/swift-account-audit
	bin/swift-account-auditor
	bin/swift-account-info
	bin/swift-account-reaper
	bin/swift-account-replicator
	bin/swift-account-server
	bin/swift-config
	bin/swift-container-auditor
	bin/swift-container-info
	bin/swift-container-replicator
	bin/swift-container-server
	bin/swift-container-sync
	bin/swift-container-updater
	bin/swift-dispersion-populate
	bin/swift-dispersion-report
	bin/swift-drive-audit
	bin/swift-form-signature
	bin/swift-get-nodes
	bin/swift-init
	bin/swift-object-auditor
	bin/swift-object-expirer
	bin/swift-object-info
	bin/swift-object-replicator
	bin/swift-object-server
	bin/swift-object-updater
	bin/swift-oldies
	bin/swift-orphans
	bin/swift-proxy-server
	bin/swift-recon
	bin/swift-recon-cron
	bin/swift-ring-builder
	bin/swift-temp-url

[entry_points]
paste.app_factory = 
	proxy = swift.proxy.server:app_factory
	object = swift.obj.server:app_factory
	mem_object = swift.obj.mem_server:app_factory
	container = swift.container.server:app_factory
	account = swift.account.server:app_factory
paste.filter_factory = 
	healthcheck = swift.common.middleware.healthcheck:filter_factory
	crossdomain = swift.common.middleware.crossdomain:filter_factory
	memcache = swift.common.middleware.memcache:filter_factory
	ratelimit = swift.common.middleware.ratelimit:filter_factory
	cname_lookup = swift.common.middleware.cname_lookup:filter_factory
	catch_errors = swift.common.middleware.catch_errors:filter_factory
	domain_remap = swift.common.middleware.domain_remap:filter_factory
	staticweb = swift.common.middleware.staticweb:filter_factory
	tempauth = swift.common.middleware.tempauth:filter_factory
	keystoneauth = swift.common.middleware.keystoneauth:filter_factory
	recon = swift.common.middleware.recon:filter_factory
	tempurl = swift.common.middleware.tempurl:filter_factory
	formpost = swift.common.middleware.formpost:filter_factory
	name_check = swift.common.middleware.name_check:filter_factory
	bulk = swift.common.middleware.bulk:filter_factory
	container_quotas = swift.common.middleware.container_quotas:filter_factory
	account_quotas = swift.common.middleware.account_quotas:filter_factory
	proxy_logging = swift.common.middleware.proxy_logging:filter_factory
	dlo = swift.common.middleware.dlo:filter_factory
	slo = swift.common.middleware.slo:filter_factory
	list_endpoints = swift.common.middleware.list_endpoints:filter_factory
	gatekeeper = swift.common.middleware.gatekeeper:filter_factory
	container_sync = swift.common.middleware.container_sync:filter_factory

[build_sphinx]
all_files = 1
build-dir = doc/build
source-dir = doc/source

[egg_info]
tag_build = 
tag_date = 0
tag_svn_revision = 0

[compile_catalog]
directory = locale
domain = swift

[update_catalog]
domain = swift
output_dir = locale
input_file = locale/swift.pot

[extract_messages]
keywords = _ l_ lazy_gettext
mapping_file = babel.cfg
output_file = locale/swift.pot

[nosetests]
exe = 1
verbosity = 2
detailed-errors = 1
cover-package = swift
cover-html = true
cover-erase = true

swift-1.13.1/.functests0000775000175400017540000000025412323703611016110 0ustar  jenkinsjenkins00000000000000#!/bin/bash

SRC_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))")

cd ${SRC_DIR}/test/functional
nosetests --exe $@
rvalue=$?
cd -

exit $rvalue
swift-1.13.1/setup.py0000664000175400017540000000143212323703614015602 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools

setuptools.setup(
    setup_requires=['pbr>=0.5.21,<1.0'],
    pbr=True)
swift-1.13.1/.unittests0000775000175400017540000000114412323703611016133 0ustar  jenkinsjenkins00000000000000#!/bin/bash

TOP_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))")

python -c 'from distutils.version import LooseVersion as Ver; import nose, sys; sys.exit(0 if Ver(nose.__version__) >= Ver("1.2.0") else 1)'
if [ $? != 0 ]; then
    cover_branches=""
else
    # Having the HTML reports is REALLY useful for achieving 100% branch
    # coverage.
    cover_branches="--cover-branches --cover-html --cover-html-dir=$TOP_DIR/cover"
fi
cd $TOP_DIR/test/unit
nosetests --exe --with-coverage --cover-package swift. --cover-erase $cover_branches $@
rvalue=$?
rm -f .coverage
cd -
exit $rvalue
swift-1.13.1/examples/0000775000175400017540000000000012323703665015714 5ustar  jenkinsjenkins00000000000000swift-1.13.1/examples/wsgi/0000775000175400017540000000000012323703665016665 5ustar  jenkinsjenkins00000000000000swift-1.13.1/examples/wsgi/account-server.wsgi.template0000664000175400017540000000101612323703611024317 0ustar  jenkinsjenkins00000000000000# Account Server wsgi Template
#
# Change %SERVICECONF% to the service conf file you are using
#
# For example:
#     Replace %SERVICECONF% by account-server/1.conf
#
# This file than need to be saved under /var/www/swift/%SERVICENAME%.wsgi
# * Replace %SERVICENAME% with the service name you use your system
#   E.g. Replace %SERVICENAME% by account-server-1

from swift.common.wsgi import init_request_processor
application, conf, logger, log_name = \
    init_request_processor('/etc/swift/%SERVICECONF%','account-server')
swift-1.13.1/examples/wsgi/object-server.wsgi.template0000664000175400017540000000101212323703611024125 0ustar  jenkinsjenkins00000000000000# Object Server wsgi Template
#
# Change %SERVICECONF% to the service conf file you are using
#
# For example:
#     Replace %SERVICECONF% by object-server/1.conf
#
# This file than need to be saved under /var/www/swift/%SERVICENAME%.wsgi
# * Replace %SERVICENAME% with the service name you use your system
#   E.g. Replace %SERVICENAME% by object-server-1

from swift.common.wsgi import init_request_processor
application, conf, logger, log_name = \
    init_request_processor('/etc/swift/%SERVICECONF%','object-server')
swift-1.13.1/examples/wsgi/proxy-server.wsgi.template0000664000175400017540000000100212323703611024037 0ustar  jenkinsjenkins00000000000000# Proxy Server wsgi Template
#
# Change %SERVICECONF% to the service conf file you are using
#
# For example:
#     Replace %SERVICECONF% by proxy-server.conf
#
# This file than need to be saved under /var/www/swift/%SERVICENAME%.wsgi
# * Replace %SERVICENAME% with the service name you use your system
#   E.g. Replace %SERVICENAME% by proxy-server

from swift.common.wsgi import init_request_processor
application, conf, logger, log_name = \
    init_request_processor('/etc/swift/%SERVICECONF%','proxy-server')
swift-1.13.1/examples/wsgi/container-server.wsgi.template0000664000175400017540000000102612323703611024646 0ustar  jenkinsjenkins00000000000000# Container Server wsgi Template
#
# Change %SERVICECONF% to the service conf file you are using
#
# For example:
#     Replace %SERVICECONF% by container-server/1.conf
#
# This file than need to be saved under /var/www/swift/%SERVICENAME%.wsgi
# * Replace %SERVICENAME% with the service name you use your system
#   E.g. Replace %SERVICENAME% by container-server-1

from swift.common.wsgi import init_request_processor
application, conf, logger, log_name = \
    init_request_processor('/etc/swift/%SERVICECONF%','container-server')
swift-1.13.1/examples/apache2/0000775000175400017540000000000012323703665017217 5ustar  jenkinsjenkins00000000000000swift-1.13.1/examples/apache2/proxy-server.template0000664000175400017540000000162512323703611023434 0ustar  jenkinsjenkins00000000000000# Proxy Server VHOST Template For Apache2
#
# Change %PORT% to the port that you wish to use on your system
# Change %SERVICENAME% to the service name you are using
# Change %USER% to the system user that will run the daemon process
# Change the debug level as you see fit
#
# For example:
#     Replace %PORT% by 8080
#     Replace %SERVICENAME% by proxy-server
#     Replace %USER% with apache (or remove it for default)

NameVirtualHost *:%PORT%
Listen %PORT%


    # The limit of an object size
    LimitRequestBody 5368709122
    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
    WSGIProcessGroup %SERVICENAME%
    WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
    WSGIApplicationGroup %{GLOBAL}
    LimitRequestFields 200
    ErrorLog /var/log/%APACHE_NAME%/%SERVICENAME%
    LogLevel debug
    CustomLog /var/log/%APACHE_NAME%/access.log combined

swift-1.13.1/examples/apache2/account-server.template0000664000175400017540000000153112323703611023703 0ustar  jenkinsjenkins00000000000000# Account Server VHOST Template For Apache2
#
# Change %PORT% to the port that you wish to use on your system
# Change %SERVICENAME% to the service name you are using
# Change %USER% to the system user that will run the daemon process
# Change the debug level as you see fit
#
# For example:
#     Replace %PORT% by 6012
#     Replace %SERVICENAME% by account-server-1
#     Replace %USER% with apache (or remove it for default)

NameVirtualHost *:%PORT%
Listen %PORT%


    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
    WSGIProcessGroup %SERVICENAME%
    WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
    WSGIApplicationGroup %{GLOBAL}
    LimitRequestFields 200
    ErrorLog /var/log/%APACHE_NAME%/%SERVICENAME%
    LogLevel debug
    CustomLog /var/log/%APACHE_NAME%/access.log combined

swift-1.13.1/examples/apache2/object-server.template0000664000175400017540000000152712323703611023522 0ustar  jenkinsjenkins00000000000000# Object Server VHOST Template For Apache2
#
# Change %PORT% to the port that you wish to use on your system
# Change %SERVICENAME% to the service name you are using
# Change %USER% to the system user that will run the daemon process
# Change the debug level as you see fit
#
# For example:
#     Replace %PORT% by 6010
#     Replace %SERVICENAME% by object-server-1
#     Replace %USER% with apache (or remove it for default)

NameVirtualHost *:%PORT%
Listen %PORT%


    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
    WSGIProcessGroup %SERVICENAME%
    WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
    WSGIApplicationGroup %{GLOBAL}
    LimitRequestFields 200
    ErrorLog /var/log/%APACHE_NAME%/%SERVICENAME%
    LogLevel debug
    CustomLog /var/log/%APACHE_NAME%/access.log combined

swift-1.13.1/examples/apache2/container-server.template0000664000175400017540000000153512323703611024235 0ustar  jenkinsjenkins00000000000000# Container Server VHOST Template For Apache2
#
# Change %PORT% to the port that you wish to use on your system
# Change %SERVICENAME% to the service name you are using
# Change %USER% to the system user that will run the daemon process
# Change the debug level as you see fit
#
# For example:
#     Replace %PORT% by 6011
#     Replace %SERVICENAME% by container-server-1
#     Replace %USER% with apache (or remove it for default)

NameVirtualHost *:%PORT%
Listen %PORT%


    WSGIDaemonProcess %SERVICENAME% processes=5 threads=1 user=%USER%
    WSGIProcessGroup %SERVICENAME%
    WSGIScriptAlias / /var/www/swift/%SERVICENAME%.wsgi
    WSGIApplicationGroup %{GLOBAL}
    LimitRequestFields 200
    ErrorLog /var/log/%APACHE_NAME%/%SERVICENAME%
    LogLevel debug
    CustomLog /var/log/%APACHE_NAME%/access.log combined

swift-1.13.1/README.md0000664000175400017540000000601712323703611015350 0ustar  jenkinsjenkins00000000000000# Swift

A distributed object storage system designed to scale from a single machine
to thousands of servers. Swift is optimized for multi-tenancy and high
concurrency. Swift is ideal for backups, web and mobile content, and any other
unstructured data that can grow without bound.

Swift provides a simple, REST-based API fully documented at
http://docs.openstack.org/.

Swift was originally developed as the basis for Rackspace's Cloud Files and
was open-sourced in 2010 as part of the OpenStack project. It has since grown
to include contributions from many companies and has spawned a thriving
ecosystem of 3rd party tools. Swift's contributors are listed in the AUTHORS
file.

## Docs

To build documentation install sphinx (`pip install sphinx`), run
`python setup.py build_sphinx`, and then browse to /doc/build/html/index.html.
These docs are auto-generated after every commit and available online at
http://docs.openstack.org/developer/swift/.

## For Developers

The best place to get started is the ["SAIO - Swift All In One"](http://docs.openstack.org/developer/swift/development_saio.html).
This document will walk you through setting up a development cluster of Swift
in a VM. The SAIO environment is ideal for running small-scale tests against
swift and trying out new features and bug fixes.

You can run unit tests with `.unittests` and functional tests with
`.functests`.

### Code Organization

 * bin/: Executable scripts that are the processes run by the deployer
 * doc/: Documentation
 * etc/: Sample config files
 * swift/: Core code
    * account/: account server
    * common/: code shared by different modules
        * middleware/: "standard", officially-supported middleware
        * ring/: code implementing Swift's ring
    * container/: container server
    * obj/: object server
    * proxy/: proxy server
 * test/: Unit and functional tests

### Data Flow

Swift is a WSGI application and uses eventlet's WSGI server. After the
processes are running, the entry point for new requests is the `Application`
class in `swift/proxy/server.py`. From there, a controller is chosen, and the
request is processed. The proxy may choose to forward the request to a back-
end server. For example, the entry point for requests to the object server is
the `ObjectController` class in `swift/obj/server.py`.


## For Deployers

Deployer docs are also available at
http://docs.openstack.org/developer/swift/. A good starting point is at
http://docs.openstack.org/developer/swift/deployment_guide.html

You can run functional tests against a swift cluster with `.functests`. These
functional tests require `/etc/swift/test.conf` to run. A sample config file
can be found in this source tree in `test/sample.conf`.

## For Client Apps

For client applications, official Python language bindings are provided at
http://github.com/openstack/python-swiftclient.

Complete API documentation at
http://docs.openstack.org/api/openstack-object-storage/1.0/content/

----

For more information come hang out in #openstack-swift on freenode.

Thanks,

The Swift Development Team
swift-1.13.1/locale/0000775000175400017540000000000012323703665015335 5ustar  jenkinsjenkins00000000000000swift-1.13.1/locale/swift.pot0000664000175400017540000005631412323703611017215 0ustar  jenkinsjenkins00000000000000# Translations template for swift.
# Copyright (C) 2011 ORGANIZATION
# This file is distributed under the same license as the swift project.
# FIRST AUTHOR , 2011.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: swift 1.2.0\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2011-01-26 23:59+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME \n"
"Language-Team: LANGUAGE \n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 0.9.4\n"

#: swift/account/auditor.py:52 swift/account/auditor.py:75
#, python-format
msgid ""
"Since %(time)s: Account audits: %(passed)s passed audit, %(failed)s "
"failed audit"
msgstr ""

#: swift/account/auditor.py:100 swift/container/auditor.py:103
#, python-format
msgid "Audit passed for %s"
msgstr ""

#: swift/account/auditor.py:103
#, python-format
msgid "ERROR Could not get account info %s"
msgstr ""

#: swift/account/reaper.py:80 swift/container/updater.py:64
#, python-format
msgid "Loading account ring from %s"
msgstr ""

#: swift/account/reaper.py:88 swift/obj/updater.py:57
#, python-format
msgid "Loading container ring from %s"
msgstr ""

#: swift/account/reaper.py:96
#, python-format
msgid "Loading object ring from %s"
msgstr ""

#: swift/account/reaper.py:106
msgid "Daemon started."
msgstr ""

#: swift/account/reaper.py:122
#, python-format
msgid "Begin devices pass: %s"
msgstr ""

#: swift/account/reaper.py:128 swift/common/utils.py:805
#: swift/obj/updater.py:74 swift/obj/updater.py:113
#, python-format
msgid "Skipping %s as it is not mounted"
msgstr ""

#: swift/account/reaper.py:132
#, python-format
msgid "Devices pass completed: %.02fs"
msgstr ""

#: swift/account/reaper.py:215
#, python-format
msgid "Beginning pass on account %s"
msgstr ""

#: swift/account/reaper.py:238
#, python-format
msgid "Exception with containers for account %s"
msgstr ""

#: swift/account/reaper.py:243
#, python-format
msgid "Exception with account %s"
msgstr ""

#: swift/account/reaper.py:244
#, python-format
msgid "Incomplete pass on account %s"
msgstr ""

#: swift/account/reaper.py:246
#, python-format
msgid ", %s containers deleted"
msgstr ""

#: swift/account/reaper.py:248
#, python-format
msgid ", %s objects deleted"
msgstr ""

#: swift/account/reaper.py:250
#, python-format
msgid ", %s containers remaining"
msgstr ""

#: swift/account/reaper.py:253
#, python-format
msgid ", %s objects remaining"
msgstr ""

#: swift/account/reaper.py:255
#, python-format
msgid ", %s containers possibly remaining"
msgstr ""

#: swift/account/reaper.py:258
#, python-format
msgid ", %s objects possibly remaining"
msgstr ""

#: swift/account/reaper.py:261
msgid ", return codes: "
msgstr ""

#: swift/account/reaper.py:265
#, python-format
msgid ", elapsed: %.02fs"
msgstr ""

#: swift/account/reaper.py:320 swift/account/reaper.py:355
#: swift/account/reaper.py:406 swift/container/updater.py:277
#, python-format
msgid "Exception with %(ip)s:%(port)s/%(device)s"
msgstr ""

#: swift/account/reaper.py:333
#, python-format
msgid "Exception with objects for container %(container)s for account %(account)s"
msgstr ""

#: swift/account/server.py:309 swift/container/server.py:397
#: swift/obj/server.py:597
#, python-format
msgid "ERROR __call__ error with %(method)s %(path)s "
msgstr ""

#: swift/auth/server.py:96 swift/common/middleware/swauth.py:94
msgid "No super_admin_key set in conf file! Exiting."
msgstr ""

#: swift/auth/server.py:152
#, python-format
msgid ""
"\n"
"THERE ARE ACCOUNTS IN YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW "
"RESELLER\n"
"PREFIX OF \"%(reseller)s\".\n"
"YOU HAVE A FEW OPTIONS:\n"
"    1. RUN \"swift-auth-update-reseller-prefixes %(db_file)s "
"%(reseller)s\",\n"
"       \"swift-init auth-server restart\", AND\n"
"       \"swift-auth-recreate-accounts -K ...\" TO CREATE FRESH ACCOUNTS.\n"
"    OR\n"
"    2. REMOVE %(db_file)s, RUN \"swift-init auth-server restart\", AND "
"RUN\n"
"       \"swift-auth-add-user ...\" TO CREATE BRAND NEW ACCOUNTS THAT WAY."
"\n"
"    OR\n"
"    3. ADD \"reseller_prefix = %(previous)s\" (WITHOUT THE QUOTES) TO "
"YOUR\n"
"       proxy-server.conf IN THE [filter:auth] SECTION AND TO YOUR\n"
"       auth-server.conf IN THE [app:auth-server] SECTION AND RUN\n"
"       \"swift-init proxy-server restart\" AND \"swift-init auth-server "
"restart\"\n"
"       TO REVERT BACK TO YOUR PREVIOUS RESELLER PREFIX.\n"
"\n"
"    %(note)s\n"
"                    "
msgstr ""

#: swift/auth/server.py:173
msgid ""
"\n"
"    SINCE YOUR PREVIOUS RESELLER PREFIX WAS AN EMPTY STRING, IT IS NOT\n"
"    RECOMMENDED TO PERFORM OPTION 3 AS THAT WOULD MAKE SUPPORTING "
"MULTIPLE\n"
"    RESELLERS MORE DIFFICULT.\n"
"                    "
msgstr ""

#: swift/auth/server.py:178
msgid "CRITICAL: "
msgstr ""

#: swift/auth/server.py:213
#, python-format
msgid "ERROR attempting to create account %(url)s: %(status)s %(reason)s"
msgstr ""

#: swift/auth/server.py:346
#, python-format
msgid ""
"ALREADY EXISTS create_user(%(account)s, %(user)s, _, %(admin)s, "
"%(reseller_admin)s) [%(elapsed).02f]"
msgstr ""

#: swift/auth/server.py:364
#, python-format
msgid ""
"FAILED create_user(%(account)s, %(user)s, _, %(admin)s, "
"%(reseller_admin)s) [%(elapsed).02f]"
msgstr ""

#: swift/auth/server.py:381
#, python-format
msgid ""
"SUCCESS create_user(%(account)s, %(user)s, _, %(admin)s, "
"%(reseller_admin)s) = %(url)s [%(elapsed).02f]"
msgstr ""

#: swift/auth/server.py:656
msgid "ERROR Unhandled exception in ReST request"
msgstr ""

#: swift/common/bench.py:85
#, python-format
msgid "%(complete)s %(title)s [%(fail)s failures], %(rate).01f/s"
msgstr ""

#: swift/common/bench.py:97
msgid "CannotSendRequest.  Skipping..."
msgstr ""

#: swift/common/bufferedhttp.py:96
#, python-format
msgid "HTTP PERF: %(time).5f seconds to %(method)s %(host)s:%(port)s %(path)s)"
msgstr ""

#: swift/common/db.py:299
msgid "Broker error trying to rollback locked connection"
msgstr ""

#: swift/common/db.py:754 swift/common/db.py:1221
#, python-format
msgid "Invalid pending entry %(file)s: %(entry)s"
msgstr ""

#: swift/common/db_replicator.py:84
#, python-format
msgid "ERROR reading HTTP response from %s"
msgstr ""

#: swift/common/db_replicator.py:123
#, python-format
msgid "Attempted to replicate %(count)d dbs in %(time).5f seconds (%(rate).5f/s)"
msgstr ""

#: swift/common/db_replicator.py:129
#, python-format
msgid "Removed %(remove)d dbs"
msgstr ""

#: swift/common/db_replicator.py:130
#, python-format
msgid "%(success)s successes, %(failure)s failures"
msgstr ""

#: swift/common/db_replicator.py:155
#, python-format
msgid "ERROR rsync failed with %(code)s: %(args)s"
msgstr ""

#: swift/common/db_replicator.py:205
#, python-format
msgid "Syncing chunks with %s"
msgstr ""

#: swift/common/db_replicator.py:213
#, python-format
msgid "ERROR Bad response %(status)s from %(host)s"
msgstr ""

#: swift/common/db_replicator.py:278
#, python-format
msgid "ERROR Unable to connect to remote server: %s"
msgstr ""

#: swift/common/db_replicator.py:316
#, python-format
msgid "Replicating db %s"
msgstr ""

#: swift/common/db_replicator.py:325 swift/common/db_replicator.py:479
#, python-format
msgid "Quarantining DB %s"
msgstr ""

#: swift/common/db_replicator.py:328
#, python-format
msgid "ERROR reading db %s"
msgstr ""

#: swift/common/db_replicator.py:361
#, python-format
msgid "ERROR Remote drive not mounted %s"
msgstr ""

#: swift/common/db_replicator.py:363
#, python-format
msgid "ERROR syncing %(file)s with node %(node)s"
msgstr ""

#: swift/common/db_replicator.py:405
msgid "ERROR Failed to get my own IPs?"
msgstr ""

#: swift/common/db_replicator.py:412
#, python-format
msgid "Skipping %(device)s as it is not mounted"
msgstr ""

#: swift/common/db_replicator.py:420
msgid "Beginning replication run"
msgstr ""

#: swift/common/db_replicator.py:425
msgid "Replication run OVER"
msgstr ""

#: swift/common/db_replicator.py:436
msgid "ERROR trying to replicate"
msgstr ""

#: swift/common/memcached.py:69
#, python-format
msgid "Timeout %(action)s to memcached: %(server)s"
msgstr ""

#: swift/common/memcached.py:72
#, python-format
msgid "Error %(action)s to memcached: %(server)s"
msgstr ""

#: swift/common/memcached.py:81
#, python-format
msgid "Error limiting server %s"
msgstr ""

#: swift/common/utils.py:88
#, python-format
msgid "Unable to locate %s in libc.  Leaving as a no-op."
msgstr ""

#: swift/common/utils.py:255
msgid "STDOUT: Connection reset by peer"
msgstr ""

#: swift/common/utils.py:257 swift/common/utils.py:260
#, python-format
msgid "STDOUT: %s"
msgstr ""

#: swift/common/utils.py:324
msgid "Connection refused"
msgstr ""

#: swift/common/utils.py:326
msgid "Host unreachable"
msgstr ""

#: swift/common/utils.py:328
msgid "Connection timeout"
msgstr ""

#: swift/common/utils.py:464
msgid "UNCAUGHT EXCEPTION"
msgstr ""

#: swift/common/utils.py:511
msgid "Error: missing config file argument"
msgstr ""

#: swift/common/utils.py:516
#, python-format
msgid "Error: unable to locate %s"
msgstr ""

#: swift/common/utils.py:743
#, python-format
msgid "Unable to read config file %s"
msgstr ""

#: swift/common/utils.py:749
#, python-format
msgid "Unable to find %s config section in %s"
msgstr ""

#: swift/common/middleware/catch_errors.py:39
#, python-format
msgid "Error: %s"
msgstr ""

#: swift/common/middleware/cname_lookup.py:91
#, python-format
msgid "Mapped %(given_domain)s to %(found_domain)s"
msgstr ""

#: swift/common/middleware/cname_lookup.py:102
#, python-format
msgid "Following CNAME chain for  %(given_domain)s to %(found_domain)s"
msgstr ""

#: swift/common/middleware/ratelimit.py:172
msgid "Returning 497 because of blacklisting"
msgstr ""

#: swift/common/middleware/ratelimit.py:185
#, python-format
msgid "Ratelimit sleep log: %(sleep)s for %(account)s/%(container)s/%(object)s"
msgstr ""

#: swift/common/middleware/ratelimit.py:192
#, python-format
msgid "Returning 498 because of ops rate limiting (Max Sleep) %s"
msgstr ""

#: swift/common/middleware/ratelimit.py:212
msgid "Warning: Cannot ratelimit without a memcached client"
msgstr ""

#: swift/common/middleware/swauth.py:635
#, python-format
msgid ""
"ERROR: Exception while trying to communicate with "
"%(scheme)s://%(host)s:%(port)s/%(path)s"
msgstr ""

#: swift/container/auditor.py:54 swift/container/auditor.py:78
#, python-format
msgid ""
"Since %(time)s: Container audits: %(pass)s passed audit, %(fail)s failed "
"audit"
msgstr ""

#: swift/container/auditor.py:68
msgid "Begin container audit \"once\" mode"
msgstr ""

#: swift/container/auditor.py:88
#, python-format
msgid "Container audit \"once\" mode completed: %.02fs"
msgstr ""

#: swift/container/auditor.py:106
#, python-format
msgid "ERROR Could not get container info %s"
msgstr ""

#: swift/container/server.py:114
#, python-format
msgid ""
"ERROR Account update failed with %(ip)s:%(port)s/%(device)s (will retry "
"later): Response %(status)s %(reason)s"
msgstr ""

#: swift/container/server.py:122
#, python-format
msgid ""
"ERROR account update failed with %(ip)s:%(port)s/%(device)s (will retry "
"later)"
msgstr ""

#: swift/container/updater.py:78 swift/obj/replicator.py:492
#, python-format
msgid "%s is not mounted"
msgstr ""

#: swift/container/updater.py:97
#, python-format
msgid "ERROR with loading suppressions from %s: "
msgstr ""

#: swift/container/updater.py:107
msgid "Begin container update sweep"
msgstr ""

#: swift/container/updater.py:140
#, python-format
msgid ""
"Container update sweep of %(path)s completed: %(elapsed).02fs, "
"%(success)s successes, %(fail)s failures, %(no_change)s with no changes"
msgstr ""

#: swift/container/updater.py:154
#, python-format
msgid "Container update sweep completed: %.02fs"
msgstr ""

#: swift/container/updater.py:164
msgid "Begin container update single threaded sweep"
msgstr ""

#: swift/container/updater.py:172
#, python-format
msgid ""
"Container update single threaded sweep completed: %(elapsed).02fs, "
"%(success)s successes, %(fail)s failures, %(no_change)s with no changes"
msgstr ""

#: swift/container/updater.py:224
#, python-format
msgid "Update report sent for %(container)s %(dbfile)s"
msgstr ""

#: swift/container/updater.py:232
#, python-format
msgid "Update report failed for %(container)s %(dbfile)s"
msgstr ""

#: swift/container/updater.py:266
#, python-format
msgid ""
"ERROR account update failed with %(ip)s:%(port)s/%(device)s (will retry "
"later): "
msgstr ""

#: swift/obj/auditor.py:61
#, python-format
msgid "Begin object audit \"%s\" mode"
msgstr ""

#: swift/obj/auditor.py:73
#, python-format
msgid ""
"Since %(start_time)s: Locally: %(passes)d passed audit, %(quars)d "
"quarantined, %(errors)d errors files/sec: %(frate).2f , bytes/sec: "
"%(brate).2f"
msgstr ""

#: swift/obj/auditor.py:90
#, python-format
msgid ""
"Object audit \"%(mode)s\" mode completed: %(elapsed).02fs. Total "
"files/sec: %(frate).2f , Total bytes/sec: %(brate).2f "
msgstr ""

#: swift/obj/auditor.py:141
#, python-format
msgid "ERROR Object %(obj)s failed audit and will be quarantined: %(err)s"
msgstr ""

#: swift/obj/auditor.py:150
#, python-format
msgid "ERROR Trying to audit %s"
msgstr ""

#: swift/obj/replicator.py:182
msgid "Error hashing suffix"
msgstr ""

#: swift/obj/replicator.py:246
#, python-format
msgid "Killing long-running rsync: %s"
msgstr ""

#: swift/obj/replicator.py:257
#, python-format
msgid "Bad rsync return code: %(args)s -> %(ret)d"
msgstr ""

#: swift/obj/replicator.py:261 swift/obj/replicator.py:265
#, python-format
msgid "Successful rsync of %(src)s at %(dst)s (%(time).03f)"
msgstr ""

#: swift/obj/replicator.py:350
#, python-format
msgid "Removing partition: %s"
msgstr ""

#: swift/obj/replicator.py:353
msgid "Error syncing handoff partition"
msgstr ""

#: swift/obj/replicator.py:383
#, python-format
msgid "%(ip)s/%(device)s responded as unmounted"
msgstr ""

#: swift/obj/replicator.py:388
#, python-format
msgid "Invalid response %(resp)s from %(ip)s"
msgstr ""

#: swift/obj/replicator.py:410
#, python-format
msgid "Error syncing with node: %s"
msgstr ""

#: swift/obj/replicator.py:414
msgid "Error syncing partition"
msgstr ""

#: swift/obj/replicator.py:424
#, python-format
msgid ""
"%(replicated)d/%(total)d (%(percentage).2f%%) partitions replicated in "
"%(time).2fs (%(rate).2f/sec, %(remaining)s remaining)"
msgstr ""

#: swift/obj/replicator.py:433
#, python-format
msgid ""
"%(checked)d suffixes checked - %(hashed).2f%% hashed, %(synced).2f%% "
"synced"
msgstr ""

#: swift/obj/replicator.py:439
#, python-format
msgid "Partition times: max %(max).4fs, min %(min).4fs, med %(med).4fs"
msgstr ""

#: swift/obj/replicator.py:446
#, python-format
msgid "Nothing replicated for %s seconds."
msgstr ""

#: swift/obj/replicator.py:475
msgid "Lockup detected.. killing live coros."
msgstr ""

#: swift/obj/replicator.py:530
msgid "Ring change detected. Aborting current replication pass."
msgstr ""

#: swift/obj/replicator.py:540
msgid "Exception in top-level replication loop"
msgstr ""

#: swift/obj/replicator.py:549
msgid "Running object replicator in script mode."
msgstr ""

#: swift/obj/replicator.py:553 swift/obj/replicator.py:565
#, python-format
msgid "Object replication complete. (%.02f minutes)"
msgstr ""

#: swift/obj/replicator.py:560
msgid "Starting object replication pass."
msgstr ""

#: swift/obj/replicator.py:566
#, python-format
msgid "Replication sleeping for %s seconds."
msgstr ""

#: swift/obj/server.py:313
#, python-format
msgid ""
"ERROR Container update failed (saving for async update later): %(status)d"
" response from %(ip)s:%(port)s/%(dev)s"
msgstr ""

#: swift/obj/server.py:319
#, python-format
msgid ""
"ERROR container update failed with %(ip)s:%(port)s/%(dev)s (saving for "
"async update later)"
msgstr ""

#: swift/obj/updater.py:65
msgid "Begin object update sweep"
msgstr ""

#: swift/obj/updater.py:89
#, python-format
msgid ""
"Object update sweep of %(device)s completed: %(elapsed).02fs, %(success)s"
" successes, %(fail)s failures"
msgstr ""

#: swift/obj/updater.py:98
#, python-format
msgid "Object update sweep completed: %.02fs"
msgstr ""

#: swift/obj/updater.py:105
msgid "Begin object update single threaded sweep"
msgstr ""

#: swift/obj/updater.py:117
#, python-format
msgid ""
"Object update single threaded sweep completed: %(elapsed).02fs, "
"%(success)s successes, %(fail)s failures"
msgstr ""

#: swift/obj/updater.py:157
#, python-format
msgid "ERROR Pickle problem, quarantining %s"
msgstr ""

#: swift/obj/updater.py:177
#, python-format
msgid "Update sent for %(obj)s %(path)s"
msgstr ""

#: swift/obj/updater.py:182
#, python-format
msgid "Update failed for %(obj)s %(path)s"
msgstr ""

#: swift/obj/updater.py:206
#, python-format
msgid "ERROR with remote server %(ip)s:%(port)s/%(device)s"
msgstr ""

#: swift/proxy/server.py:165 swift/proxy/server.py:629
#: swift/proxy/server.py:696 swift/proxy/server.py:712
#: swift/proxy/server.py:721 swift/proxy/server.py:1004
#: swift/proxy/server.py:1044 swift/proxy/server.py:1089
msgid "Object"
msgstr ""

#: swift/proxy/server.py:170
#, python-format
msgid "Could not load object segment %(path)s: %(status)s"
msgstr ""

#: swift/proxy/server.py:177 swift/proxy/server.py:210
#: swift/proxy/server.py:257
#, python-format
msgid "ERROR: While processing manifest /%(acc)s/%(cont)s/%(obj)s"
msgstr ""

#: swift/proxy/server.py:292
#, python-format
msgid "%(msg)s %(ip)s:%(port)s"
msgstr ""

#: swift/proxy/server.py:304
#, python-format
msgid "ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: %(info)s"
msgstr ""

#: swift/proxy/server.py:328
#, python-format
msgid "Node error limited %(ip)s:%(port)s (%(device)s)"
msgstr ""

#: swift/proxy/server.py:388 swift/proxy/server.py:1451
#: swift/proxy/server.py:1497 swift/proxy/server.py:1545
#: swift/proxy/server.py:1590
msgid "Account"
msgstr ""

#: swift/proxy/server.py:389
#, python-format
msgid "Trying to get account info for %s"
msgstr ""

#: swift/proxy/server.py:466 swift/proxy/server.py:740
#: swift/proxy/server.py:772 swift/proxy/server.py:1214
#: swift/proxy/server.py:1301 swift/proxy/server.py:1356
#: swift/proxy/server.py:1413
msgid "Container"
msgstr ""

#: swift/proxy/server.py:467
#, python-format
msgid "Trying to get container info for %s"
msgstr ""

#: swift/proxy/server.py:552
#, python-format
msgid "%(type)s returning 503 for %(statuses)s"
msgstr ""

#: swift/proxy/server.py:598 swift/proxy/server.py:697
#, python-format
msgid "Trying to %(method)s %(path)s"
msgstr ""

#: swift/proxy/server.py:627
msgid "Client disconnected on read"
msgstr ""

#: swift/proxy/server.py:630
#, python-format
msgid "Trying to read during GET of %s"
msgstr ""

#: swift/proxy/server.py:653
#, python-format
msgid "ERROR %(status)d %(body)s From %(type)s Server"
msgstr ""

#: swift/proxy/server.py:692
#, python-format
msgid "ERROR %(status)d %(body)s From Object Server"
msgstr ""

#: swift/proxy/server.py:776 swift/proxy/server.py:783
#, python-format
msgid "Object manifest GET could not continue listing: %s %s"
msgstr ""

#: swift/proxy/server.py:905
msgid "Object POST"
msgstr ""

#: swift/proxy/server.py:1005
#, python-format
msgid "Expect: 100-continue on %s"
msgstr ""

#: swift/proxy/server.py:1017
#, python-format
msgid "Object PUT returning 503, %(conns)s/%(nodes)s required connections"
msgstr ""

#: swift/proxy/server.py:1045
#, python-format
msgid "Trying to write to %s"
msgstr ""

#: swift/proxy/server.py:1049
#, python-format
msgid ""
"Object PUT exceptions during send, %(conns)s/%(nodes)s required "
"connections"
msgstr ""

#: swift/proxy/server.py:1058
#, python-format
msgid "ERROR Client read timeout (%ss)"
msgstr ""

#: swift/proxy/server.py:1063
msgid "ERROR Exception causing client disconnect"
msgstr ""

#: swift/proxy/server.py:1068
msgid "Client disconnected without sending enough data"
msgstr ""

#: swift/proxy/server.py:1083
#, python-format
msgid "ERROR %(status)d %(body)s From Object Server re: %(path)s"
msgstr ""

#: swift/proxy/server.py:1090
#, python-format
msgid "Trying to get final status of PUT to %s"
msgstr ""

#: swift/proxy/server.py:1093
#, python-format
msgid "Object servers returned %s mismatched etags"
msgstr ""

#: swift/proxy/server.py:1101
msgid "Object PUT"
msgstr ""

#: swift/proxy/server.py:1153
msgid "Object DELETE"
msgstr ""

#: swift/proxy/server.py:1302 swift/proxy/server.py:1498
#, python-format
msgid "Trying to PUT to %s"
msgstr ""

#: swift/proxy/server.py:1314
msgid "Container PUT"
msgstr ""

#: swift/proxy/server.py:1357 swift/proxy/server.py:1546
#, python-format
msgid "Trying to POST %s"
msgstr ""

#: swift/proxy/server.py:1369
msgid "Container POST"
msgstr ""

#: swift/proxy/server.py:1414 swift/proxy/server.py:1591
#, python-format
msgid "Trying to DELETE %s"
msgstr ""

#: swift/proxy/server.py:1426
msgid "Container DELETE"
msgstr ""

#: swift/proxy/server.py:1433
msgid "Returning 503 because not all container nodes confirmed DELETE"
msgstr ""

#: swift/proxy/server.py:1508
msgid "Account PUT"
msgstr ""

#: swift/proxy/server.py:1556
msgid "Account POST"
msgstr ""

#: swift/proxy/server.py:1601
msgid "Account DELETE"
msgstr ""

#: swift/proxy/server.py:1757
msgid "ERROR Unhandled exception in request"
msgstr ""

#: swift/stats/access_processor.py:63 swift/stats/stats_processor.py:40
#, python-format
msgid "Bad line data: %s"
msgstr ""

#: swift/stats/access_processor.py:67
#, python-format
msgid "Bad server name: found \"%(found)s\" expected \"%(expected)s\""
msgstr ""

#: swift/stats/access_processor.py:75
#, python-format
msgid "Invalid path: %(error)s from data: %(log)s"
msgstr ""

#: swift/stats/access_processor.py:199
#, python-format
msgid "I found a bunch of bad lines in %(name)s (%(bad)d bad, %(total)d total)"
msgstr ""

#: swift/stats/account_stats.py:55
msgid "Gathering account stats"
msgstr ""

#: swift/stats/account_stats.py:59
#, python-format
msgid "Gathering account stats complete (%0.2f minutes)"
msgstr ""

#: swift/stats/account_stats.py:75
#, python-format
msgid "Device %s is not mounted, skipping."
msgstr ""

#: swift/stats/account_stats.py:81
#, python-format
msgid "Path %s does not exist, skipping."
msgstr ""

#: swift/stats/log_processor.py:62
#, python-format
msgid "Loaded plugin \"%s\""
msgstr ""

#: swift/stats/log_processor.py:79
#, python-format
msgid "Processing %(obj)s with plugin \"%(plugin)s\""
msgstr ""

#: swift/stats/log_processor.py:179
#, python-format
msgid "Bad compressed data for %s"
msgstr ""

#: swift/stats/log_processor.py:240
msgid "Beginning log processing"
msgstr ""

#: swift/stats/log_processor.py:278
#, python-format
msgid "found %d processed files"
msgstr ""

#: swift/stats/log_processor.py:283
#, python-format
msgid "loaded %d files to process"
msgstr ""

#: swift/stats/log_processor.py:286 swift/stats/log_processor.py:360
#, python-format
msgid "Log processing done (%0.2f minutes)"
msgstr ""

#: swift/stats/log_uploader.py:71
msgid "Uploading logs"
msgstr ""

#: swift/stats/log_uploader.py:74
#, python-format
msgid "Uploading logs complete (%0.2f minutes)"
msgstr ""

#: swift/stats/log_uploader.py:129
#, python-format
msgid "Unexpected log: %s"
msgstr ""

#: swift/stats/log_uploader.py:135
#, python-format
msgid "Skipping log: %(file)s (< %(cutoff)d seconds old)"
msgstr ""

#: swift/stats/log_uploader.py:142
#, python-format
msgid "Log %s is 0 length, skipping"
msgstr ""

#: swift/stats/log_uploader.py:144
#, python-format
msgid "Processing log: %s"
msgstr ""

#: swift/stats/log_uploader.py:165
#, python-format
msgid "Uploaded log %(file)s to %(target)s"
msgstr ""

#: swift/stats/log_uploader.py:170
#, python-format
msgid "ERROR: Upload of log %s failed!"
msgstr ""

swift-1.13.1/PKG-INFO0000664000175400017540000001062512323703665015177 0ustar  jenkinsjenkins00000000000000Metadata-Version: 1.1
Name: swift
Version: 1.13.1
Summary: OpenStack Object Storage
Home-page: http://www.openstack.org/
Author: OpenStack
Author-email: openstack-dev@lists.openstack.org
License: UNKNOWN
Description: # Swift
        
        A distributed object storage system designed to scale from a single machine
        to thousands of servers. Swift is optimized for multi-tenancy and high
        concurrency. Swift is ideal for backups, web and mobile content, and any other
        unstructured data that can grow without bound.
        
        Swift provides a simple, REST-based API fully documented at
        http://docs.openstack.org/.
        
        Swift was originally developed as the basis for Rackspace's Cloud Files and
        was open-sourced in 2010 as part of the OpenStack project. It has since grown
        to include contributions from many companies and has spawned a thriving
        ecosystem of 3rd party tools. Swift's contributors are listed in the AUTHORS
        file.
        
        ## Docs
        
        To build documentation install sphinx (`pip install sphinx`), run
        `python setup.py build_sphinx`, and then browse to /doc/build/html/index.html.
        These docs are auto-generated after every commit and available online at
        http://docs.openstack.org/developer/swift/.
        
        ## For Developers
        
        The best place to get started is the ["SAIO - Swift All In One"](http://docs.openstack.org/developer/swift/development_saio.html).
        This document will walk you through setting up a development cluster of Swift
        in a VM. The SAIO environment is ideal for running small-scale tests against
        swift and trying out new features and bug fixes.
        
        You can run unit tests with `.unittests` and functional tests with
        `.functests`.
        
        ### Code Organization
        
         * bin/: Executable scripts that are the processes run by the deployer
         * doc/: Documentation
         * etc/: Sample config files
         * swift/: Core code
            * account/: account server
            * common/: code shared by different modules
                * middleware/: "standard", officially-supported middleware
                * ring/: code implementing Swift's ring
            * container/: container server
            * obj/: object server
            * proxy/: proxy server
         * test/: Unit and functional tests
        
        ### Data Flow
        
        Swift is a WSGI application and uses eventlet's WSGI server. After the
        processes are running, the entry point for new requests is the `Application`
        class in `swift/proxy/server.py`. From there, a controller is chosen, and the
        request is processed. The proxy may choose to forward the request to a back-
        end server. For example, the entry point for requests to the object server is
        the `ObjectController` class in `swift/obj/server.py`.
        
        
        ## For Deployers
        
        Deployer docs are also available at
        http://docs.openstack.org/developer/swift/. A good starting point is at
        http://docs.openstack.org/developer/swift/deployment_guide.html
        
        You can run functional tests against a swift cluster with `.functests`. These
        functional tests require `/etc/swift/test.conf` to run. A sample config file
        can be found in this source tree in `test/sample.conf`.
        
        ## For Client Apps
        
        For client applications, official Python language bindings are provided at
        http://github.com/openstack/python-swiftclient.
        
        Complete API documentation at
        http://docs.openstack.org/api/openstack-object-storage/1.0/content/
        
        ----
        
        For more information come hang out in #openstack-swift on freenode.
        
        Thanks,
        
        The Swift Development Team
        
        
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Environment :: OpenStack
Classifier: Intended Audience :: Information Technology
Classifier: Intended Audience :: System Administrators
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Operating System :: POSIX :: Linux
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.6
Classifier: Programming Language :: Python :: 2.7
swift-1.13.1/CHANGELOG0000664000175400017540000011135212323703611015302 0ustar  jenkinsjenkins00000000000000swift (1.13.1)

    * Change the behavior of CORS responses to better match the spec

      A new proxy config variable (strict_cors_mode, default to True)
      has been added. Setting it to False keeps the old behavior. For
      an overview of old versus new behavior, please see
      https://review.openstack.org/#/c/69419/

    * Invert the responsibility of the two instances of proxy-logging in
      the proxy pipeline

      The first proxy_logging middleware instance to receive a request
      in the pipeline marks that request as handling it. So now, the
      left most proxy_logging middleware handles logging for all
      client requests, and the right most proxy_logging middleware
      handles all other requests initiated from within the pipeline to
      its left. This fixes logging related to large object
      requests not properly recording bandwidth.

    * Added swift-container-info and swift-account-info tools

    * Allow specification of object devices for audit

    * Dynamic large object COPY requests with ?multipart-manifest=get
      now work as expected

    * When a client is downloading a large object and one of the segment
      reads gets bad data, Swift will now immediately abort the request.

    * Fix ring-builder crash when a ring partition was assigned to a
      deleted device, zero-weighted device, and normal device

    * Make probetests work with conf.d configs

    * Various other minor bug fixes and improvements.

swift (1.13.0)

    * Account-level ACLs and ACL format v2

      Accounts now have a new privileged header to represent ACLs or
      any other form of account-level access control. The value of
      the header is a JSON dictionary string to be interpreted by the
      auth system. A reference implementation is given in TempAuth.
      Please see the full docs at
      http://swift.openstack.org/overview_auth.html

    * Added a WSGI environment flag to stop swob from always using
      absolute location. This is useful if middleware needs to use
      out-of-spec Location headers in a response.

    * Container sync proxies now support simple load balancing

    * Config option to lower the timeout for recoverable object GETs

    * Add a way to ratelimit all writes to an account

    * Allow multiple storage_domain values in cname_lookup middleware

    * Moved all DLO functionality into middleware

      The proxy will automatically insert the dlo middleware at an
      appropriate place in the pipeline the same way it does with the
      gatekeeper middleware. Clusters will still support DLOs after upgrade
      even with an old config file that doesn't mention dlo at all.

    * Remove python-swiftclient dependency

    * Add secondary groups to process user during privilege escalation

    * When logging request headers, it is now possible to specify
      specifically which headers should be logged

    * Added log_requests config parameter to account and container servers
      to match the parameter in the object server. This allows a deployer
      to turn off log messages for these processes.

    * Ensure swift.source is set for DLO/SLO requests

    * Fixed an issue where overwriting segments in a dynamic manifest
      could cause issues on pipelined requests.

    * Properly handle COPY verb in container quota middleware

    * Improved StaticWeb 404 error message on web-listings and index

    * Various other minor bug fixes and improvements.

swift (1.12.0)

    * Several important pieces of information have been added to /info:

       - Configured constraints are included and allow a client to discover
         the limits on names and object sizes that the cluster supports.

       - The supported tempurl methods are now included.

       - Static large object constraints are now included.

    * The Last-Modified header value returned will now be the object's
      timestamp rounded up to the next second. This allows subsequent
      requests with If-[un]modified-Since to use the Last-Modified
      value as expected.

    * Non-integer values for if-delete-at headers will now properly
      report a 400 error instead of a 503.

    * Fix object versioning with non-ASCII container names.

    * Bulk delete with POST now works properly.

    * Generic means for persisting system metadata

      Swift now supports system-level metadata on accounts and
      containers. System metadata provides a means to store internal
      custom metadata with associated Swift resources in a safe and
      secure fashion without actually having to plumb custom metadata
      through the core swift servers. The new gatekeeper middleware
      prevents this system metadata from leaking into the request or
      being set by a client.

    * catch_errors and gatekeeper middleware are now forced into the proxy
      pipeline if not explicitly referenced.

    * New container sync configuration option, separating the end user
      from knowing the required end point and adding more secure
      signed requests. See
      http://swift.openstack.org/overview_container_sync.html for full
      information.

    * bulk middleware now can be configured to retry deleting containers.

    * The default yield_frequency used to keep client connections alive
      during slow bulk requests was reduced from 60 seconds to 10 seconds.
      While this is a change to a default, it should not affect deployments
      and there is no migration process needed.

    * Swift processes will attempt to set RLIMIT_NPROC to 8192.

    * Server processes will now exit with a non-zero error code on config
      errors.

    * Warn if read_affinity is configured but not enabled.

    * Fix checkmount error parsing in swift-recon.

    * Log at warn level when an object is quarantined.

    * Fixed CVE-2014-0006 to avoid a potential timing attack with tempurl.

    * Various other minor bug fixes and improvements.


swift (1.11.0)

    * Added discoverable capabilities

      A Swift proxy server now by default (although it can be turned off)
      will respond to requests to /info. The response to these requests
      include information about the cluster and can be used by clients to
      determine which features are supported in the cluster.

    * Object replication ssync (an rsync alternative)

      A Swift storage node can now be configured to use Swift primitives
      for replication transport instead of rsync. This is an experimental
      feature that is not yet considered production ready.

    * If a source times out on an object server read, try another one
      of them with a modified range.

    * The proxy now responds to many types of requests as soon as it
      has a quorum. This can help speed up responses (without
      changing the results), especially when one node is acting up.
      There is a post_quorum_timeout config value that can tune how
      long to wait for requests to finish after a quorum has been
      established.

    * Add accurate timestamps in proxy log lines for the start and
      end of a request. These are added as new fields on the end of
      the existing log lines, and therefore should not break
      existing, well-behaved log processors.

    * Add an "inline" query parameter to tempurl

      By default, temporary URLs add a "Content-Disposition" header
      that forces many clients to download the object. Now, temporary
      URLs support an optional "inline" query parameter that will
      force a "Content-Disposition: inline" header to be added to the
      response, overriding the default.

    * Use TCP_NODELAY for created sockets. This can dramatically
      lower latency for small object workloads.

    * DiskFile API, with reference implementation

      The DiskFile abstraction for talking to data on disk has been
      refactored to allow alternate implementations to be developed.
      Included in the codebase is an in-memory reference
      implementation. For full documentation, please see the developer
      documentation. The DiskFile API is still a work in progress and
      is not yet finalized.

    * Removal of swift-bench

      The included benchmarking tool swift-bench has been extracted
      from the codebase and is now in its own repository at
      https://github.com/openstack/swift-bench. New swift-bench
      binaries and packages may be found on PyPI at
      https://pypi.python.org/pypi/swift-bench

    * Bulk delete now also supports the POST verb, in addition to DELETE

    * Added functionality to the swift-ring-builder to support
      limited recreation of ring builder files from the ring file itself.

    * HEAD on account now returns 410 if account was deleted and
      not yet reaped. The old behavior was to return a 404.

    * Fixed a bug introduced since the 1.10.0 release that
      prevented expired objects from being removed from the system.
      This resulted in orphaned expired objects taking up space on
      the system but inaccessible to the API. This regression and
      fix are only important if you have deployed code since the
      1.10.0 release. For a full discussion, including a script that
      can be used to clean up orphaned objects, see
      https://bugs.launchpad.net/swift/+bug/1257330

    * Tie socket write buffer size to server chunk size parameter. This
      pairs the underlying network buffer size with the size of data
      that Swift attempts to read from the connection, thereby
      improving efficiency and throughput on connections.

    * Fix 500 from account-quota middleware. If a user had set
      X-Account-Meta-Quota-Bytes to something non-integer prior to
      the installation of the account-quota middleware, then the
      quota check would choke on it. Now a non-integer value is
      treated as "no quota".

    * Quarantine objects with busted metadata. Before, if you
      encountered an object with corrupt or missing xattrs, the
      object server would return a 500 on GET, and wouldn't quarantine
      anything. Now the object server returns a 404 for that GET and
      the corrupted file is quarantined, thus giving replication a
      chance to fix it.

    * Fix quarantine and error counts in audit logs

    * Report transaction ID in failure exception logs

    * Make pbr a build-time only dependency

    * Worked around a bug in eventlet 0.9.16 where the size of the
      memcache connection pools would grow unbounded.

    * Tempurl keys are now properly stored as utf8

    * Fixed an issue where concurrent PUT requests to accounts or
      containers may result in errors due to locked databases.

    * Handle copy requests in account and container quota middleware

    * Now ensure that a WWW-Authenticate header is on all 401 responses

    * Various other bug fixes and improvements

swift (1.10.0)

    * Added support for pooling memcache connections

    * Added support to replicating handoff partitions first in object
      replication. Can also configure how many remote nodes a storage node
      must talk to before removing a local handoff partition.

    * Fixed bug where memcache entries would not expire

    * Much faster calculation for choosing handoff nodes

    * Added container listing ratelimiting

    * Fixed issue where the proxy would continue to read from a storage
      server even after a client had disconnected

    * Added support for headers that are only visible to the owner of a Swift
      account

    * Fixed ranged GET with If-None-Match

    * Fixed an issue where rings may not be balanced after initial creation

    * Fixed internationalization support

    * Return the correct etag for a static large object on the PUT response

    * Allow users to extract archives to containers with ACLs set

    * Fix support for range requests against static large objects

    * Now logs x-copy-from header in a useful place

    * Reverted back to old XML output of account and container listings to
      ensure older clients do not break

    * Account quotas now appropriately handle copy requests

    * Fix issue with UTF-8 handling in versioned writes

    * Various other bug fixes and improvements, including support for running
      Swift under Pypy and continuing work to support storage policies

swift (1.9.1)

    * Disallow PUT, POST, and DELETE requests from creating older tombstone
      files, preventing the possibility of filling up the disk and removing
      unnecessary container updates.

    * Set default wsgi workers to cpu_count

      Change the default value of wsgi workers from 1 to auto. The new
      default value for workers in the proxy, container, account & object
      wsgi servers will spawn as many workers per process as you have cpu
      cores. This will not be ideal for some configurations, but it's much
      more likely to produce a successful out of the box deployment.

    * Added reveal_sensitive_prefix config setting to filter the auth token
      logged by the proxy server.

    * Ensure Keystone's reseller prefix ends with an underscore. Previously
      this was a recommendation--now it is enforced.

    * Added log_file_pattern config to swift-drive-audit for drive errors

    * Add support for telling Swift to detect a content type on a request.

    * Additional object stats are now logged in the object auditor

    * Moved the DiskFile interface into its own module

    * Ensure the SQLite cursors are closed when creating functions

    * Better support for valid Accept headers

    * In Keystone, don't allow users to delete their own account

    * Return a UTC timezone designator in container listings

    * Ensure that users can't remove their account quotas

    * Allow floating point value for dispersion coverage

    * Fix incorrect error page handling in staticweb

    * Add utf-8 charset to multipart-manifest=get response.

    * Allow dispersion tools to use keystone server with insecure certificate

    * Ensure that files are always closed in tests

    * Use OpenStack's "Hacking" guidelines for code formatting

    * Various other minor bug fixes and improvements

swift (1.9.0)

    * Global clusters support

      The "region" concept introduced in Swift 1.8.0 has been augmented with
      support for using a separate replication network and configuring read
      and write affinity. These features combine to offer support for a single
      Swift cluster spanning wide geographic area.

    * Disk performance

      The object server now can be configured to use threadpools to increase
      performance and smooth out latency throughout the system. Also, many
      disk operations were reordered to increase reliability and improve
      performance.

    * Added config file conf.d support

      Allow Swift daemons and servers to optionally accept a directory as the
      configuration parameter. This allows different parts of the config file
      to be managed separately, eg each middleware could use a separate file
      for its particular config settings.

    * Allow two TempURL keys per account

      By adding a second key, a user can safely rotate keys and prevent URLs
      already in use from becoming invalid. TempURL middlware has also been
      updated to allow a configuable set of allowed methods and to prevent a
      bugrelated to content-disposition names.

    * Added crossdomain.xml middleware. See
      http://docs.openstack.org/developer/swift/crossdomain.html for details

    * Added rsync bandwidth limit setting for object replicator

    * Transaction ID updated to include the time and an optional suffix

    * Added x-remove-versions-location header to disable versioned writes

    * Improvements to support for Keystone ACLs

    * Added parallelism to object expirer daemon

    * Added support for ring hash prefix in addition to the existing suffix

    * Allow all headers requested for CORS

    * Stop getting useless bytes on manifest Range requests

    * Improved container-sync resiliency

    * Added example Apache config files. See
      http://docs.openstack.org/developer/swift/apache_deployment_guide.html
      for more info

    * If an account is marked as deleted but hasn't been reaped and is still
      on disk, responses will include an "X-Account-Status" header

    * Fix 503 on account/container HEAD with invalid format

    * Added extra safety on account-level DELETE when using bulk deletes

    * Made colons quote-safe in logs (mainly for IPv6)

    * Fixed bug with bulk delete max items

    * Fixed static large object manifest range requests

    * Prevent static large objects from containing other static large objects

    * Fixed issue with use of delimiter in container queries where some
      objects would not be listed

    * Various other minor bug fixes and improvements

swift (1.8.0)

    * Make rings' replica count adjustable

    * Added a region tier to the ring above zones

    * Added timing-based sorting of object servers on read requests

    * Added support for auto-extract archive uploads

    * Added support for bulk delete requests

    * Added support for large objects with static manifests

    * Added list_endpoints middleware to provide an API for determining where
      the ring places data

    * proxy-logging middleware can now handle logging for other middleware

      proxy-logging should be used twice in the proxy pipeline. The first
      handles middleware logs for requests that never made it all the way
      to the server. The last handles requests that do make it to the server.

      This is a change that may require an update to your proxy server
      config file or custom middleware that you may be using. See the full
      docs at http://docs.openstack.org/developer/swift/misc.html#module-swift.common.middleware.proxy_logging.

    * Changed the default sample rate for a few high-traffic requests.

      Added log_statsd_sample_rate_factor to globally tune the StatsD
      sample rate. This tunable can be used to reduce StatsD traffic
      proportionally for all metrics and is intended to replace
      log_statsd_default_sample_rate, which is left alone for
      backward-compatibility, should anyone be using it.

    * Added swift_hash_path_prefix option to swift.conf

      New deployments are advised to set this value to a random secret
      to protect against hash collisions

    * Added user-managed container quotas

    * Added support for account-level quotas managed by an auth reseller

    * Added --run-dir option to swift-init

    * Added more options to swift-bench

    * Added support for CORS "actual requests"

    * Added fallocate_reserve option to protect against full drives

    * Allow ring rebalance to take a seed

    * Ring serialization will now produce the same gzip file (Py2.7)

    * Added support to swift-drive-audit for handling rotated logs

    * Added first-byte latency timings for GET requests

    * Added per disk PUT timing monitoring support

    * Added speed limit options for DB auditor

    * Force log entries to be one line

    * Ensure that fsync is used and not just fdatasync

    * Improved handoff node selection

    * Deprecated keystone is_admin feature

    * Fix large objects with unicode in the segment names

    * Update Swift's MemcacheRing to provide API compatibility with
      standard Python memcache libraries

    * Various other minor bug fixes and improvements

swift (1.7.6)

    * Better tempauth storage URL guessing

    * Added --top option to swift-recon -d

    * Allow optional, temporary healthcheck failure

    * keystoneauth middleware now supports cross-tenant ACLs

    * Add dispersion report flags to limit reports

    * Add config option to turn eventlet debug on/off

    * Added override option for swift-init's KILL_WAIT

    * Added oldest and most recent replication pass to swift-recon

    * Fixed 500 error response when GETing a many-segment manifest

    * Memcached keys now use a delta timeout when possible

    * Refactor DiskFile to hide temp file names and exts

    * Remove IP-based container-sync ACLs from auth middlewares

    * Fixed bug in deleting memcached account info data

    * Fixed lazy-listing of object manifest segments

    * Fixed bug where a ? in the object name caused an error

    * Swift now returns 406 if it can't satisfy Accept

    * Fix infinite recursion bug in object replicator

    * Swift will now reject names with NULL characters

    * Fixed object-auditor logging to use a minimum of unix sockets

    * Various other minor bug fixes and improvements

swift (1.7.5)

    * Support OPTIONS verb, including CORS preflight requests

    * Added support for custom log handlers

    * Range support is extended to support GET requests with multiple ranges.
      Multi-range GETs are not yet supported against large-object manifests.

    * Cluster constraints are now settable by config

    * Replicators can now run against specific devices or partitions

    * swift-bench now supports running on multiple cores and multiple servers

    * Added partition option to swift-get-nodes

    * Allow underscores in account and user in tempauth via base64 encodings

    * New option to the dispersion report to output the missing partitions

    * Changed storage server StatsD metrics to report timings instead of
      counts for errors. See the admin guide for the updated metric names.

    * Removed a dependency on WebOb and replaced it with an internal module

    * Fixed config parsing in swift-bench -x

    * Fixed sample_rate in StatsD logging

    * Track unlinks of async_pendings with StatsD

    * Remove double GET on range requests

    * Allow unsetting of X-Container-Sync-To and ACL headers

    * DB reclamation now removes empty suffix directories

    * Fix non-standard 100-continue behavior

    * Allow object-expirer to delete the last copy of a versioned object

    * Only set TCP_KEEPIDLE on systems where it is supported

    * Fix stdin flush and fdatasync issues on BSD platforms

    * Allow object-expirer to delete the last version of an object

    * Various other minor bug fixes and improvements

swift (1.7.4)

    * Fix issue where early client disconnects may have caused a memory leak

swift (1.7.2)

    * Fix issue where memcache serialization was not properly loading
      the config value

swift (1.7.0)

    * Use custom encoding for ring data instead of pickle

      Serialize RingData in a versioned, custom format which is a combination
      of a JSON-encoded header and .tostring() dumps of the
      replica2part2dev_id arrays.  This format deserializes hundreds of times
      faster than rings serialized with Python 2.7's pickle (a significant
      performance regression for ring loading between Python 2.6 and Python
      2.7).  Fixes bug 1031954.

      The new implementation is backward-compatible; if a ring
      does not begin with a new-style magic string, it is assumed to be an
      old-style pickle-dumped ring and is handled as before.  So new Swift
      code can read old rings, but old Swift code will not be able to read
      newly-serialized rings.

    * Do not use pickle for serialization in memcache, but JSON

      To avoid issues on upgrades (unability to read pickled values, and cache
      poisoning for old servers not understanding JSON), we add a
      memcache_serialization_support configuration option, with the following
      values:

       0 = older, insecure pickle serialization
       1 = json serialization but pickles can still be read (still insecure)
       2 = json serialization only (secure and the default)

      To avoid an instant full cache flush, existing installations should
      upgrade with 0, then set to 1 and reload, then after some time (24
      hours) set to 2 and reload. Support for 0 and 1 will be removed in
      future versions.

    * Update proxy-server StatsD logging. This is a significant change to the
      existing StatsD intigration. Docs for this feature can be found in
      doc/source/admin_guide.rst.

    * Improved swift-bench to allow random object sizes and better usability

    * Updated probe tests

    * Replicator removal metrics are now generated on a per-device basis

    * Made object replicator locking more optimistic

    * Split proxy-server code into separate modules

    * Fixed bug where swift-recon would not report all unmounted drives

    * Fixed issue where a LockTimeout may have caused a file descriptor to
      not be closed properly

    * Fixed a bug where an error may have caused the proxy to stop returning
      data to a client

    * Fixed bug where expirer would get confused by odd deletion times

    * Fixed a bug where auto-creating accounts would return an error if they
      were recreated after being deleted

    * Fix when rate_limit_after_segment kicks in

    * fallocate() failures properly return HTTPInsufficientStorage from
      object-server before reading from wsgi.input, allowing the proxy
      server to quickly error_limit that node

    * Fixed error with large object manifests and x-newest headers on GET

    * Various other minor bug fixes and improvements

swift (1.6.0)

    * Removed bin/swift and swift/common/client.py from the swift repo. These
      tools are now managed in the python-swiftclient project. The
      python-swiftclient project is a second deliverable of the openstack
      swift project.

    * Moved swift_auth (openstack keystone) middleware from keystone project
      into swift project

    * Made dispersion report work with any replica count other than 3. This
      substantially affects the JSON output of the dispersion report, and any
      tools written to consume this output will need to be updated.

    * Added Solaris (Illumos) compatibility

    * Added -a option to swift-get-nodes to show all handoffs

    * Add UDP protocol support for logger

    * Added config options for rate limiting of large object downloads.

    * Added config option `log_handoffs` (defaults to True) to proxy server
      to log and update statsd with information about when a handoff node is
      used. This is helpful to track the health of the cluster.

    * swift-bench can now use auth 2.0

    * Support forbidding substrings based on a regexp in name_filter
      middleware

    * Hardened internal server processes so only authorized methods can be
      called.

    * Made ranged requests on large objects work correctly when size of
      manifest file is not 0 byte

    * Added option to dispersion report to print 404s to stdout

    * Fix object replication on older rsync versions when using ipv4

    * Fixed bug with container reclaim/report race

    * Make object server's caching more configurable.

    * Check disk failure before syncing for each partition

    * Allow special characters to be referenced by manifest objects

    * Validate devices and partitions to avoid directory traversals

    * Support WebOb 1.2

    * Ensure that accessing the ring devs reloads the ring if necessary.
      Specifically, this allows replication to work when it has been started
      with an empty ring.

    * Various other minor bug fixes and improvements

swift (1.5.0)

    * New option to toggle SQLite database preallocation with account
      and container servers.

      IMPORTANT:
      The default for database preallocation is now off when before
      it was always on. This will affect performance on clusters that
      use standard drives with shared account, container, object
      servers. Such deployments will need to update their
      configurations to turn database preallocation back on (see
      account-server.conf-sample and container-server.conf.sample
      files).

      If you are using dedicated account and container servers with
      SSDs, you should defragment your file systems after upgrade and
      should notice dramatically less disk usage.

    * swift3 middleware removed and moved to http://github.com/fujita/swift3.
      This will require a config change in the proxy server and adds a new
      dependency for deployers using this middleware.

    * Moved proxy server logging to middleware. This requires a config change
      in the proxy server.

    * Added object versioning feature. (See docs for full description)

    * Add statsd logging throughout the system (beta, some event names may
      change)

    * Expanded swift-recon middleware support

    * The ring builder now supports as-unique-as-possible partition
      placement, unified balancing methods, and can work on more than one
      device at a time.

    * Numerous bug fixes to StaticWeb (previously unusable at scale).

    * Bug fixes to all middleware to allow passthrough requests under various
      conditions and to share pre-authed request code (which previously had
      differing behaviors and interaction bugs).

    * Bug fix to object expirer that could cause infinite looping.

    * Added optional delay to account reaping.

    * Async-pending write optimization.

    * Dispersion tools now support multiple auth versions

    * Updated man pages

    * Proxy server can now deny requests to particular hostnames

    * Updated docs for domain remap middleware

    * Updated docs for cname lookup middleware

    * Made swift CLI binary easier to wrap

    * Proxy will now also return X-Timestamp header

    * Added associated projects doc as a place to track ecosystem projects

    * end_marker made consistent across both object and container listings

    * Various other minor bug fixes and improvements

swift (1.4.8)

    * Added optional max_containers_per_account restriction

    * Added alternate metadata header removal method

    * Added optional name_check middleware filter

    * Added support for venv-based test runs with tox

    * StaticWeb behavior change with X-Web-Mode: true and
      non-StaticWeb-enabled containers (immediately 404s instead of passing
      the request on down the WSGI pipeline).

    * Fixed typo in swift-dispersion-report JSON output.

    * Swift-Recon-related fix to create temporary files on the same disk as
      their final destinations.

    * Updated return codes in swift3 middleware

    * Fixed swift3 middleware to allow Content-Range header in response

    * Updated swift.common.client and swift CLI tool with auth 2.0 changes

    * Swift CLI tool now supports common openstack auth args

    * Body of HTTP responses now included in error messages of swift CLI tool

    * Refactored some ring building functions for clarity and simplicity

swift (1.4.7)

    * Improvements to account and container replication.

    * Fix for account servers allowing .pending to exist before .db.

    * Fixed possible key-guessing exploit in formpost.

    * Fixed bug in ring builder when removing a large percentage of devices.

    * Swift CLI tool now supports openstack-standard CLI flags.

    * New JSON output option for swift-dispersion-report.

    * Removed old stats tools.

    * Other bug fixes and documentation updates.

swift (1.4.6)

    * TempURL and FormPost middleware added

    * Added memcache.conf option

    * Dropped eval-based json parser fallback

    * Properly lose all groups when dropping privileges

    * Fix permissions when creating files

    * Fixed bug regarding negative Content-Length in requests

    * Consistent formatting on Last-Modified response header

    * Added timeout option to swift-recon

    * Allow arguments to be passed to nosetest

    * Removed tools/rfc.sh

    * Other minor bug fixes

swift (1.4.5)

    * New swift-orphans and swift-oldies command line tools to detect
      orphaned Swift processes and long running processes.

    * Command line tool "swift" now supports marker queries.

    * StaticWeb middleware improved to save an extra request when
      possible.

    * Updated swift-init to support swift-object-expirer.

    * Fixed object replicator timeout handling [bug 814263].

    * Fixed accept header 503 vs. 400 [bug 891247].

    * More exception handling for auditors.

    * Doc updates for PPA [bug 905608].

    * Doc updates to explain replication more clearly [bug 906976].

    * Updated SAIO instructions to no longer mention ~/swift/trunk.

    * Fixed docstrings in the ring code.

    * PEP8 Updates.

swift (1.4.4)

    * Fixes to prevent socket hoarding (memory leak)

    * Add sockstat info to recon.

    * Fixed leak from SegmentedIterable.

    * Fixed bufferedhttp to deref socks and fps.

    * Add support for OS Auth API version 2.

    * Make Eventlet's WSGI server log differently.

    * Updated TimeoutError and except Exception refs.

    * Fixed time-sensitive tests.

    * Fixed object manifest etags.

    * Fixes for swift-recon disk usage distribution graph.

    * Adding new manpages for configuration files.

    * Change bzr to swift in getting_started doc.

    * Fixes the HTTPConflict import.

    * Expiring Objects Support.

    * Fixing bug with x-trans-id.

    * Requote the source when doing a COPY.

    * Add documentation for Swift Recon.

    * Make drive audit regexes detect 4-letter drives.

    * Adding what acc/cont/obj into the ratelimit error messages.

    * Query only specific zone via swift-recon.

swift (1.4.3)

    * Additional quarantine catching code.

    * Added client_ip to all proxy log lines not otherwise containing it.

    * Content-Type is now application/xml for "GET services/bucket" swift3
      middleware requests.

    * Alpha release of the Swift Recon Experiment

    * Fix last modified date for swift3 middleware.

    * Fix to clear account/container metadata on account/container deletion.

    * Fix for corner case regarding X-Newest.

    * Fix for object auditor running out of file descriptors.

    * Fix to return all proper headers for manifest objects.

    * Fix to the swift tool to strip any leading slashes on file names when
      uploading.

swift (1.4.2)

    * Removed stats/logging code from Swift [now in separate slogging project].

    * Container Synchronization Feature - First Edition

    * Fix swift3 authentication bug about the Date and X-Amz-Date handling.

    * Changing ratelimiting so that it only limits PUTs/DELETEs.

    * Object POSTs are implemented as COPYs now by default (you can revert to
      previous implementation with conf object_post_as_copy = false)

    * You can specify X-Newest: true on GETs and HEADs to indicate you want
      Swift to query all backend copies and return the newest version
      retrieved.

    * Object COPY requests now always copy the newest object they can find.

    * Account and container GETs and HEADs now shuffle the nodes they use to
      balance load.

    * Fixed the infinite charset: utf-8 bug

    * This fixes the bug that drop_buffer_cache() doesn't work on systems where
      off_t isn't 64 bits.

swift (1.4.1)

    * st renamed to swift

    * swauth was separated froms swift. It is now its own project and can be
      found at https://github.com/gholt/swauth.

    * tempauth middleware added as an extremely limited auth system for dev
      work.

    * Account and container listings now properly labeled UTF-8 (previously the
      label was "utf8").

    * Accounts are auto-created if an auth token is valid when the
      account_autocreate proxy config parameter is set to true.

swift (1.4.0)

    * swift-bench now cleans up containers it creates.

    * WSGI servers now load WSGI filters and applications after forking for
      better plugin support.

    * swauth-cleanup-tokens now handles 404s on token containers and tokens
      better.

    * Proxy logs the remote IP address as the client IP in the absence of
      X-Forwarded-For and X-Cluster-Client-IP headers instead of - like it did
      before.

    * Swift3 WSGI middleware added support for param-signed URLs.

    * swauth- scripts now exit with proper exit codes.

    * Fixed a bug where allowed_headers weren't honored for HEAD requests.

    * Double quarantining of corrupted sqlite3 databases now works.

    * Fix for Object replicator breaking when running object replicator with no
      objects on the server.

    * Added the Accept-Ranges header to GET and HEAD requests.

    * When a single object has multiple async pending updates on a single
      device, only latest async pending is now sent.

    * Fixed issue of Swift3 WSGI middleware not working correctly with '/' in
      object names.

    * Renamed swift-stats-* to swift-dispersion-* to avoid confusion with log
      stats stuff.

    * Added X-Trans-Id transaction id header to every response.

    * Fixed a Python 2.7 compatibility problem.

    * Now using bracketed notation for ip literals in rsync calls, so
      compressed ipv6 literals work.

    * Added a container stats collector and refactoring some of the stats code.

    * Changed subdir nodes in XML formatted object listings to align with
      object nodes. Now: foo Before:
      .

    * Fixed bug in Swauth to support for multiple swauth instances.

    * swift-ring-builder: Added list_parts command which shows common
      partitions for a given list of devices.

    * Object auditor now shows better statistics updates in the logs.

    * Stats uploaders now allow overrides for source_filename_pattern and
      new_log_cutoff values.
swift-1.13.1/bin/0000775000175400017540000000000012323703665014646 5ustar  jenkinsjenkins00000000000000swift-1.13.1/bin/swift-ring-builder0000775000175400017540000000134412323703611020302 0ustar  jenkinsjenkins00000000000000#!/usr/bin/python
# Copyright (c) 2014 Christian Schwede 
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import sys

from swift.cli.ringbuilder import main


if __name__ == "__main__":
    sys.exit(main())
swift-1.13.1/bin/swift-account-auditor0000775000175400017540000000156512323703611021025 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.account.auditor import AccountAuditor
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(AccountAuditor, conf_file, **options)
swift-1.13.1/bin/swift-container-server0000775000175400017540000000156712323703611021214 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi

if __name__ == '__main__':
    conf_file, options = parse_options()
    sys.exit(run_wsgi(conf_file,
                      'container-server', default_port=6001, **options))
swift-1.13.1/bin/swift-orphans0000775000175400017540000001147212323703611017374 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import optparse
import os
import signal
import subprocess
import sys

from swift.common.manager import RUN_DIR

if __name__ == '__main__':
    parser = optparse.OptionParser(usage='''%prog [options]

Lists and optionally kills orphaned Swift processes. This is done by scanning
/var/run/swift for .pid files and listing any processes that look like Swift
processes but aren't associated with the pids in those .pid files. Any Swift
processes running with the 'once' parameter are ignored, as those are usually
for full-speed audit scans and such.

Example (sends SIGTERM to all orphaned Swift processes older than two hours):
%prog -a 2 -k TERM
        '''.strip())
    parser.add_option('-a', '--age', dest='hours', type='int', default=24,
                      help="look for processes at least HOURS old; "
                      "default: 24")
    parser.add_option('-k', '--kill', dest='signal',
                      help='send SIGNAL to matched processes; default: just '
                      'list process information')
    parser.add_option('-w', '--wide', dest='wide', default=False,
                      action='store_true',
                      help="don't clip the listing at 80 characters")
    parser.add_option('-r', '--run-dir', type="str",
                      dest="run_dir", default=RUN_DIR,
                      help="alternative directory to store running pid files "
                      "default: %s" % RUN_DIR)
    (options, args) = parser.parse_args()

    pids = []

    for root, directories, files in os.walk(options.run_dir):
        for name in files:
            if name.endswith('.pid'):
                pids.append(open(os.path.join(root, name)).read().strip())
                pids.extend(subprocess.Popen(
                    ['ps', '--ppid', pids[-1], '-o', 'pid', '--no-headers'],
                    stdout=subprocess.PIPE).communicate()[0].split())

    listing = []
    for line in subprocess.Popen(
            ['ps', '-eo', 'etime,pid,args', '--no-headers'],
            stdout=subprocess.PIPE).communicate()[0].split('\n'):
        if not line:
            continue
        hours = 0
        try:
            etime, pid, args = line.split(None, 2)
        except ValueError:
            sys.exit('Could not process ps line %r' % line)
        if pid in pids:
            continue
        if (not args.startswith('/usr/bin/python /usr/bin/swift-') and
            not args.startswith('/usr/bin/python /usr/local/bin/swift-')) or \
                'swift-orphans' in args or \
                'once' in args.split():
            continue
        args = args.split('-', 1)[1]
        etime = etime.split('-')
        if len(etime) == 2:
            hours = int(etime[0]) * 24
            etime = etime[1]
        elif len(etime) == 1:
            etime = etime[0]
        else:
            sys.exit('Could not process etime value from %r' % line)
        etime = etime.split(':')
        if len(etime) == 3:
            hours += int(etime[0])
        elif len(etime) != 2:
            sys.exit('Could not process etime value from %r' % line)
        if hours >= options.hours:
            listing.append((str(hours), pid, args))

    if not listing:
        exit()

    hours_len = len('Hours')
    pid_len = len('PID')
    args_len = len('Command')
    for hours, pid, args in listing:
        hours_len = max(hours_len, len(hours))
        pid_len = max(pid_len, len(pid))
        args_len = max(args_len, len(args))
    args_len = min(args_len, 78 - hours_len - pid_len)

    print ('%%%ds %%%ds %%s' % (hours_len, pid_len)) % \
          ('Hours', 'PID', 'Command')
    for hours, pid, args in listing:
        print ('%%%ds %%%ds %%s' % (hours_len, pid_len)) % \
              (hours, pid, args[:args_len])

    if options.signal:
        try:
            signum = int(options.signal)
        except ValueError:
            signum = getattr(signal, options.signal.upper(),
                             getattr(signal, 'SIG' + options.signal.upper(),
                                     None))
        if not signum:
            sys.exit('Could not translate %r to a signal number.' %
                     options.signal)
        print 'Sending processes %s (%d) signal...' % (options.signal, signum),
        for hours, pid, args in listing:
            os.kill(int(pid), signum)
        print 'Done.'
swift-1.13.1/bin/swift-object-auditor0000775000175400017540000000231112323703611020625 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.obj.auditor import ObjectAuditor
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
from optparse import OptionParser

if __name__ == '__main__':
    parser = OptionParser("%prog CONFIG [options]")
    parser.add_option('-z', '--zero_byte_fps',
                      help='Audit only zero byte files at specified files/sec')
    parser.add_option('-d', '--devices',
                      help='Audit only given devices. Comma-separated list')
    conf_file, options = parse_options(parser=parser, once=True)
    run_daemon(ObjectAuditor, conf_file, **options)
swift-1.13.1/bin/swift-object-replicator0000775000175400017540000000242312323703611021326 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.obj.replicator import ObjectReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon
from optparse import OptionParser

if __name__ == '__main__':
    parser = OptionParser("%prog CONFIG [options]")
    parser.add_option('-d', '--devices',
                      help='Replicate only given devices. '
                           'Comma-separated list')
    parser.add_option('-p', '--partitions',
                      help='Replicate only given partitions. '
                           'Comma-separated list')
    conf_file, options = parse_options(parser=parser, once=True)
    run_daemon(ObjectReplicator, conf_file, **options)
swift-1.13.1/bin/swift-drive-audit0000775000175400017540000001560612323703611020142 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import glob
import os
import re
import subprocess
import sys
from ConfigParser import ConfigParser

from swift.common.utils import backward, get_logger


def get_devices(device_dir, logger):
    devices = []
    for line in open('/proc/mounts').readlines():
        data = line.strip().split()
        block_device = data[0]
        mount_point = data[1]
        if mount_point.startswith(device_dir):
            device = {}
            device['mount_point'] = mount_point
            device['block_device'] = block_device
            try:
                device_num = os.stat(block_device).st_rdev
            except OSError:
                # If we can't stat the device, then something weird is going on
                logger.error("Error: Could not stat %s!" %
                             block_device)
                continue
            device['major'] = str(os.major(device_num))
            device['minor'] = str(os.minor(device_num))
            devices.append(device)
    for line in open('/proc/partitions').readlines()[2:]:
        major, minor, blocks, kernel_device = line.strip().split()
        device = [d for d in devices
                  if d['major'] == major and d['minor'] == minor]
        if device:
            device[0]['kernel_device'] = kernel_device
    return devices


def get_errors(error_re, log_file_pattern, minutes):
    # Assuming log rotation is being used, we need to examine
    # recently rotated files in case the rotation occurred
    # just before the script is being run - the data we are
    # looking for may have rotated.
    #
    # The globbing used before would not work with all out-of-box
    # distro setup for logrotate and syslog therefore moving this
    # to the config where one can set it with the desired
    # globbing pattern.
    log_files = [f for f in glob.glob(log_file_pattern)]
    log_files.sort()

    now_time = datetime.datetime.now()
    end_time = now_time - datetime.timedelta(minutes=minutes)
    # kern.log does not contain the year so we need to keep
    # track of the year and month in case the year recently
    # ticked over
    year = now_time.year
    prev_entry_month = now_time.month
    errors = {}

    reached_old_logs = False
    for path in log_files:
        try:
            f = open(path)
        except IOError:
            logger.error("Error: Unable to open " + path)
            print("Unable to open " + path)
            sys.exit(1)
        for line in backward(f):
            if '[    0.000000]' in line \
                or 'KERNEL supported cpus:' in line \
                    or 'BIOS-provided physical RAM map:' in line:
                # Ignore anything before the last boot
                reached_old_logs = True
                break
            # Solves the problem with year change - kern.log does not
            # keep track of the year.
            log_time_entry = line.split()[:3]
            if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
                year -= 1
            prev_entry_month = log_time_entry[0]
            log_time_string = '%s %s' % (year, ' '.join(log_time_entry))
            try:
                log_time = datetime.datetime.strptime(
                    log_time_string, '%Y %b %d %H:%M:%S')
            except ValueError:
                continue
            if log_time > end_time:
                for err in error_re:
                    for device in err.findall(line):
                        errors[device] = errors.get(device, 0) + 1
            else:
                reached_old_logs = True
                break
        if reached_old_logs:
            break
    return errors


def comment_fstab(mount_point):
    with open('/etc/fstab', 'r') as fstab:
        with open('/etc/fstab.new', 'w') as new_fstab:
            for line in fstab:
                parts = line.split()
                if len(parts) > 2 and line.split()[1] == mount_point:
                    new_fstab.write('#' + line)
                else:
                    new_fstab.write(line)
    os.rename('/etc/fstab.new', '/etc/fstab')


if __name__ == '__main__':
    c = ConfigParser()
    try:
        conf_path = sys.argv[1]
    except Exception:
        print "Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1]
        sys.exit(1)
    if not c.read(conf_path):
        print "Unable to read config file %s" % conf_path
        sys.exit(1)
    conf = dict(c.items('drive-audit'))
    device_dir = conf.get('device_dir', '/srv/node')
    minutes = int(conf.get('minutes', 60))
    error_limit = int(conf.get('error_limit', 1))
    log_file_pattern = conf.get('log_file_pattern',
                                '/var/log/kern.*[!.][!g][!z]')
    error_re = []
    for conf_key in conf:
        if conf_key.startswith('regex_pattern_'):
            error_pattern = conf[conf_key]
            try:
                r = re.compile(error_pattern)
            except re.error:
                sys.exit('Error: unable to compile regex pattern "%s"' %
                         error_pattern)
            error_re.append(r)
    if not error_re:
        error_re = [
            re.compile(r'\berror\b.*\b(sd[a-z]{1,2}\d?)\b'),
            re.compile(r'\b(sd[a-z]{1,2}\d?)\b.*\berror\b'),
        ]
    conf['log_name'] = conf.get('log_name', 'drive-audit')
    logger = get_logger(conf, log_route='drive-audit')
    devices = get_devices(device_dir, logger)
    logger.debug("Devices found: %s" % str(devices))
    if not devices:
        logger.error("Error: No devices found!")
    errors = get_errors(error_re, log_file_pattern, minutes)
    logger.debug("Errors found: %s" % str(errors))
    unmounts = 0
    for kernel_device, count in errors.items():
        if count >= error_limit:
            device = \
                [d for d in devices if d['kernel_device'] == kernel_device]
            if device:
                mount_point = device[0]['mount_point']
                if mount_point.startswith(device_dir):
                    logger.info("Unmounting %s with %d errors" %
                                (mount_point, count))
                    subprocess.call(['umount', '-fl', mount_point])
                    logger.info("Commenting out %s from /etc/fstab" %
                                (mount_point))
                    comment_fstab(mount_point)
                    unmounts += 1
    if unmounts == 0:
        logger.info("No drives were unmounted")
swift-1.13.1/bin/swift-container-updater0000775000175400017540000000157312323703611021347 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.container.updater import ContainerUpdater
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(ContainerUpdater, conf_file, **options)
swift-1.13.1/bin/swift-oldies0000775000175400017540000000524012323703611017175 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import optparse
import subprocess
import sys


if __name__ == '__main__':
    parser = optparse.OptionParser(usage='''%prog [options]

Lists old Swift processes.
        '''.strip())
    parser.add_option('-a', '--age', dest='hours', type='int', default=720,
                      help='look for processes at least HOURS old; '
                      'default: 720 (30 days)')
    (options, args) = parser.parse_args()

    listing = []
    for line in subprocess.Popen(
            ['ps', '-eo', 'etime,pid,args', '--no-headers'],
            stdout=subprocess.PIPE).communicate()[0].split('\n'):
        if not line:
            continue
        hours = 0
        try:
            etime, pid, args = line.split(None, 2)
        except ValueError:
            sys.exit('Could not process ps line %r' % line)
        if not args.startswith('/usr/bin/python /usr/bin/swift-') and \
                not args.startswith('/usr/bin/python /usr/local/bin/swift-'):
            continue
        args = args.split('-', 1)[1]
        etime = etime.split('-')
        if len(etime) == 2:
            hours = int(etime[0]) * 24
            etime = etime[1]
        elif len(etime) == 1:
            etime = etime[0]
        else:
            sys.exit('Could not process etime value from %r' % line)
        etime = etime.split(':')
        if len(etime) == 3:
            hours += int(etime[0])
        elif len(etime) != 2:
            sys.exit('Could not process etime value from %r' % line)
        if hours >= options.hours:
            listing.append((str(hours), pid, args))

    if not listing:
        exit()

    hours_len = len('Hours')
    pid_len = len('PID')
    args_len = len('Command')
    for hours, pid, args in listing:
        hours_len = max(hours_len, len(hours))
        pid_len = max(pid_len, len(pid))
        args_len = max(args_len, len(args))
    args_len = min(args_len, 78 - hours_len - pid_len)

    print ('%%%ds %%%ds %%s' % (hours_len, pid_len)) % \
          ('Hours', 'PID', 'Command')
    for hours, pid, args in listing:
        print ('%%%ds %%%ds %%s' % (hours_len, pid_len)) % \
              (hours, pid, args[:args_len])
swift-1.13.1/bin/swift-account-audit0000775000175400017540000003733512323703611020470 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from urllib import quote
from hashlib import md5
import getopt
from itertools import chain

import simplejson
from eventlet.greenpool import GreenPool
from eventlet.event import Event

from swift.common.ring import Ring
from swift.common.utils import split_path
from swift.common.bufferedhttp import http_connect


usage = """
Usage!

%(cmd)s [options] [url 1] [url 2] ...
    -c [concurrency]      Set the concurrency, default 50
    -r [ring dir]         Ring locations, default /etc/swift
    -e [filename]         File for writing a list of inconsistent urls
    -d                    Also download files and verify md5

You can also feed a list of urls to the script through stdin.

Examples!

    %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076
    %(cmd)s SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container/object
    %(cmd)s -e errors.txt SOSO_88ad0b83-b2c5-4fa1-b2d6-60c597202076/container
    %(cmd)s < errors.txt
    %(cmd)s -c 25 -d < errors.txt
""" % {'cmd': sys.argv[0]}


class Auditor(object):
    def __init__(self, swift_dir='/etc/swift', concurrency=50, deep=False,
                 error_file=None):
        self.pool = GreenPool(concurrency)
        self.object_ring = Ring(swift_dir, ring_name='object')
        self.container_ring = Ring(swift_dir, ring_name='container')
        self.account_ring = Ring(swift_dir, ring_name='account')
        self.deep = deep
        self.error_file = error_file
        # zero out stats
        self.accounts_checked = self.account_exceptions = \
            self.account_not_found = self.account_container_mismatch = \
            self.account_object_mismatch = self.objects_checked = \
            self.object_exceptions = self.object_not_found = \
            self.object_checksum_mismatch = self.containers_checked = \
            self.container_exceptions = self.container_count_mismatch = \
            self.container_not_found = self.container_obj_mismatch = 0
        self.list_cache = {}
        self.in_progress = {}

    def audit_object(self, account, container, name):
        path = '/%s/%s/%s' % (account, container, name)
        part, nodes = self.object_ring.get_nodes(
            account, container.encode('utf-8'), name.encode('utf-8'))
        container_listing = self.audit_container(account, container)
        consistent = True
        if name not in container_listing:
            print "  Object %s missing in container listing!" % path
            consistent = False
            hash = None
        else:
            hash = container_listing[name]['hash']
        etags = []
        for node in nodes:
            try:
                if self.deep:
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], part, 'GET', path, {})
                    resp = conn.getresponse()
                    calc_hash = md5()
                    chunk = True
                    while chunk:
                        chunk = resp.read(8192)
                        calc_hash.update(chunk)
                    calc_hash = calc_hash.hexdigest()
                    if resp.status // 100 != 2:
                        self.object_not_found += 1
                        consistent = False
                        print '  Bad status GETting object "%s" on %s/%s' \
                            % (path, node['ip'], node['device'])
                        continue
                    if resp.getheader('ETag').strip('"') != calc_hash:
                        self.object_checksum_mismatch += 1
                        consistent = False
                        print '  MD5 does not match etag for "%s" on %s/%s' \
                            % (path, node['ip'], node['device'])
                    etags.append(resp.getheader('ETag'))
                else:
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], part, 'HEAD',
                                        path.encode('utf-8'), {})
                    resp = conn.getresponse()
                    if resp.status // 100 != 2:
                        self.object_not_found += 1
                        consistent = False
                        print '  Bad status HEADing object "%s" on %s/%s' \
                            % (path, node['ip'], node['device'])
                        continue
                    etags.append(resp.getheader('ETag'))
            except Exception:
                self.object_exceptions += 1
                consistent = False
                print '  Exception fetching object "%s" on %s/%s' \
                    % (path, node['ip'], node['device'])
                continue
        if not etags:
            consistent = False
            print "  Failed fo fetch object %s at all!" % path
        elif hash:
            for etag in etags:
                if resp.getheader('ETag').strip('"') != hash:
                    consistent = False
                    self.object_checksum_mismatch += 1
                    print '  ETag mismatch for "%s" on %s/%s' \
                        % (path, node['ip'], node['device'])
        if not consistent and self.error_file:
            print >>open(self.error_file, 'a'), path
        self.objects_checked += 1

    def audit_container(self, account, name, recurse=False):
        if (account, name) in self.in_progress:
            self.in_progress[(account, name)].wait()
        if (account, name) in self.list_cache:
            return self.list_cache[(account, name)]
        self.in_progress[(account, name)] = Event()
        print 'Auditing container "%s"' % name
        path = '/%s/%s' % (account, name)
        account_listing = self.audit_account(account)
        consistent = True
        if name not in account_listing:
            consistent = False
            print "  Container %s not in account listing!" % path
        part, nodes = \
            self.container_ring.get_nodes(account, name.encode('utf-8'))
        rec_d = {}
        responses = {}
        for node in nodes:
            marker = ''
            results = True
            while results:
                try:
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], part, 'GET',
                                        path.encode('utf-8'), {},
                                        'format=json&marker=%s' %
                                        quote(marker.encode('utf-8')))
                    resp = conn.getresponse()
                    if resp.status // 100 != 2:
                        self.container_not_found += 1
                        consistent = False
                        print('  Bad status GETting container "%s" on %s/%s' %
                              (path, node['ip'], node['device']))
                        break
                    if node['id'] not in responses:
                        responses[node['id']] = dict(resp.getheaders())
                    results = simplejson.loads(resp.read())
                except Exception:
                    self.container_exceptions += 1
                    consistent = False
                    print '  Exception GETting container "%s" on %s/%s' % \
                        (path, node['ip'], node['device'])
                    break
                if results:
                    marker = results[-1]['name']
                    for obj in results:
                        obj_name = obj['name']
                        if obj_name not in rec_d:
                            rec_d[obj_name] = obj
                        if (obj['last_modified'] !=
                                rec_d[obj_name]['last_modified']):
                            self.container_obj_mismatch += 1
                            consistent = False
                            print("  Different versions of %s/%s "
                                  "in container dbs." % (name, obj['name']))
                            if (obj['last_modified'] >
                                    rec_d[obj_name]['last_modified']):
                                rec_d[obj_name] = obj
        obj_counts = [int(header['x-container-object-count'])
                      for header in responses.values()]
        if not obj_counts:
            consistent = False
            print "  Failed to fetch container %s at all!" % path
        else:
            if len(set(obj_counts)) != 1:
                self.container_count_mismatch += 1
                consistent = False
                print "  Container databases don't agree on number of objects."
                print "  Max: %s, Min: %s" % (max(obj_counts), min(obj_counts))
        self.containers_checked += 1
        self.list_cache[(account, name)] = rec_d
        self.in_progress[(account, name)].send(True)
        del self.in_progress[(account, name)]
        if recurse:
            for obj in rec_d.keys():
                self.pool.spawn_n(self.audit_object, account, name, obj)
        if not consistent and self.error_file:
            print >>open(self.error_file, 'a'), path
        return rec_d

    def audit_account(self, account, recurse=False):
        if account in self.in_progress:
            self.in_progress[account].wait()
        if account in self.list_cache:
            return self.list_cache[account]
        self.in_progress[account] = Event()
        print 'Auditing account "%s"' % account
        consistent = True
        path = '/%s' % account
        part, nodes = self.account_ring.get_nodes(account)
        responses = {}
        for node in nodes:
            marker = ''
            results = True
            while results:
                node_id = node['id']
                try:
                    conn = http_connect(node['ip'], node['port'],
                                        node['device'], part, 'GET', path, {},
                                        'format=json&marker=%s' %
                                        quote(marker.encode('utf-8')))
                    resp = conn.getresponse()
                    if resp.status // 100 != 2:
                        self.account_not_found += 1
                        consistent = False
                        print("  Bad status GETting account '%s' "
                              " from %ss:%ss" %
                              (account, node['ip'], node['device']))
                        break
                    results = simplejson.loads(resp.read())
                except Exception:
                    self.account_exceptions += 1
                    consistent = False
                    print("  Exception GETting account '%s' on %ss:%ss" %
                          (account, node['ip'], node['device']))
                    break
                if node_id not in responses:
                    responses[node_id] = [dict(resp.getheaders()), []]
                responses[node_id][1].extend(results)
                if results:
                    marker = results[-1]['name']
        headers = [resp[0] for resp in responses.values()]
        cont_counts = [int(header['x-account-container-count'])
                       for header in headers]
        if len(set(cont_counts)) != 1:
            self.account_container_mismatch += 1
            consistent = False
            print("  Account databases for '%s' don't agree on"
                  " number of containers." % account)
            if cont_counts:
                print "  Max: %s, Min: %s" % (max(cont_counts),
                                              min(cont_counts))
        obj_counts = [int(header['x-account-object-count'])
                      for header in headers]
        if len(set(obj_counts)) != 1:
            self.account_object_mismatch += 1
            consistent = False
            print("  Account databases for '%s' don't agree on"
                  " number of objects." % account)
            if obj_counts:
                print "  Max: %s, Min: %s" % (max(obj_counts),
                                              min(obj_counts))
        containers = set()
        for resp in responses.values():
            containers.update(container['name'] for container in resp[1])
        self.list_cache[account] = containers
        self.in_progress[account].send(True)
        del self.in_progress[account]
        self.accounts_checked += 1
        if recurse:
            for container in containers:
                self.pool.spawn_n(self.audit_container, account,
                                  container, True)
        if not consistent and self.error_file:
            print >>open(self.error_file, 'a'), path
        return containers

    def audit(self, account, container=None, obj=None):
        if obj and container:
            self.pool.spawn_n(self.audit_object, account, container, obj)
        elif container:
            self.pool.spawn_n(self.audit_container, account, container, True)
        else:
            self.pool.spawn_n(self.audit_account, account, True)

    def wait(self):
        self.pool.waitall()

    def print_stats(self):

        def _print_stat(name, stat):
            # Right align stat name in a field of 18 characters
            print "{0:>18}: {1}".format(name, stat)

        print
        _print_stat("Accounts checked", self.accounts_checked)
        if self.account_not_found:
            _print_stat("Missing Replicas", self.account_not_found)
        if self.account_exceptions:
            _print_stat("Exceptions", self.account_exceptions)
        if self.account_container_mismatch:
            _print_stat("Container mismatch", self.account_container_mismatch)
        if self.account_object_mismatch:
            _print_stat("Object mismatch", self.account_object_mismatch)
        print
        _print_stat("Containers checked", self.containers_checked)
        if self.container_not_found:
            _print_stat("Missing Replicas", self.container_not_found)
        if self.container_exceptions:
            _print_stat("Exceptions", self.container_exceptions)
        if self.container_count_mismatch:
            _print_stat("Count mismatch", self.container_count_mismatch)
        if self.container_obj_mismatch:
            _print_stat("Object mismatch", self.container_obj_mismatch)
        print
        _print_stat("Objects checked", self.objects_checked)
        if self.object_not_found:
            _print_stat("Missing Replicas", self.object_not_found)
        if self.object_exceptions:
            _print_stat("Exceptions", self.object_exceptions)
        if self.object_checksum_mismatch:
            _print_stat("MD5 Mismatch", self.object_checksum_mismatch)


if __name__ == '__main__':
    try:
        optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
    except getopt.GetoptError as err:
        print str(err)
        print usage
        sys.exit(2)
    if not args and os.isatty(sys.stdin.fileno()):
        print usage
        sys.exit()
    opts = dict(optlist)
    options = {
        'concurrency': int(opts.get('-c', 50)),
        'error_file': opts.get('-e', None),
        'swift_dir': opts.get('-r', '/etc/swift'),
        'deep': '-d' in opts,
    }
    auditor = Auditor(**options)
    if not os.isatty(sys.stdin.fileno()):
        args = chain(args, sys.stdin)
    for path in args:
        path = '/' + path.rstrip('\r\n').lstrip('/')
        auditor.audit(*split_path(path, 1, 3, True))
    auditor.wait()
    auditor.print_stats()
swift-1.13.1/bin/swift-recon0000775000175400017540000000133612323703611017026 0ustar  jenkinsjenkins00000000000000#!/usr/bin/python
# Copyright (c) 2014 Christian Schwede 
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import sys

from swift.cli.recon import main


if __name__ == "__main__":
    sys.exit(main())
swift-1.13.1/bin/swift-object-info0000775000175400017540000001023412323703611020114 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from datetime import datetime
from hashlib import md5
from optparse import OptionParser

from swift.common.ring import Ring
from swift.obj.diskfile import read_metadata
from swift.common.utils import hash_path, storage_directory


def print_object_info(datafile, check_etag=True, swift_dir='/etc/swift'):
    if not os.path.exists(datafile) or not datafile.endswith('.data'):
        print "Data file doesn't exist"
        sys.exit(1)
    try:
        ring = Ring(swift_dir, ring_name='object')
    except Exception:
        ring = None
    fp = open(datafile, 'rb')
    metadata = read_metadata(fp)
    path = metadata.pop('name', '')
    content_type = metadata.pop('Content-Type', '')
    ts = metadata.pop('X-Timestamp', '')
    etag = metadata.pop('ETag', '')
    length = metadata.pop('Content-Length', '')
    if path:
        print 'Path: %s' % path
        account, container, obj = path.split('/', 3)[1:]
        print '  Account: %s' % account
        print '  Container: %s' % container
        print '  Object: %s' % obj
        obj_hash = hash_path(account, container, obj)
        print '  Object hash: %s' % obj_hash
    else:
        print 'Path: Not found in metadata'
    if content_type:
        print 'Content-Type: %s' % content_type
    else:
        print 'Content-Type: Not found in metadata'
    if ts:
        print 'Timestamp: %s (%s)' % (datetime.fromtimestamp(float(ts)), ts)
    else:
        print 'Timestamp: Not found in metadata'

    file_len = None
    if check_etag:
        h = md5()
        file_len = 0
        while True:
            data = fp.read(64 * 1024)
            if not data:
                break
            h.update(data)
            file_len += len(data)
        h = h.hexdigest()
        if etag:
            if h == etag:
                print 'ETag: %s (valid)' % etag
            else:
                print "Etag: %s doesn't match file hash of %s!" % (etag, h)
        else:
            print 'ETag: Not found in metadata'
    else:
        print 'ETag: %s (not checked)' % etag
        file_len = os.fstat(fp.fileno()).st_size

    if length:
        if file_len == int(length):
            print 'Content-Length: %s (valid)' % length
        else:
            print "Content-Length: %s doesn't match file length of %s" % (
                length, file_len)
    else:
        print 'Content-Length: Not found in metadata'
    print 'User Metadata: %s' % metadata
    if ring is not None:
        print 'Ring locations:'
        part, nodes = ring.get_nodes(account, container, obj)
        for node in nodes:
            print ('  %s:%s - /srv/node/%s/%s/%s.data' %
                   (node['ip'], node['port'], node['device'],
                    storage_directory('objects', part, obj_hash), ts))
        print
        print 'note: /srv/node is used as default value of `devices`, '\
              'the real value is set in object-server.conf '\
              'on each storage node.'
    fp.close()


if __name__ == '__main__':
    parser = OptionParser()
    parser.set_defaults(check_etag=True, swift_dir='/etc/swift')
    parser.add_option(
        '-n', '--no-check-etag',
        action="store_false", dest="check_etag",
        help="Don't verify file contents against stored etag")
    parser.add_option(
        '-d', '--swift-dir',
        help="Pass location of swift directory")

    options, args = parser.parse_args()

    if len(args) < 1:
        print "Usage: %s [-n] [-d] OBJECT_FILE" % sys.argv[0]
        sys.exit(1)

    print_object_info(args[0], check_etag=options.check_etag,
                      swift_dir=options.swift_dir)
swift-1.13.1/bin/swift-form-signature0000775000175400017540000000660512323703611020666 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hmac
from hashlib import sha1
from os.path import basename
from sys import argv, exit
from time import time


if __name__ == '__main__':
    if len(argv) != 7:
        prog = basename(argv[0])
        print 'Syntax: %s    ' \
              '  ' % prog
        print
        print 'Where:'
        print '              The prefix to use for form uploaded'
        print '                    objects. For example:'
        print '                    /v1/account/container/object_prefix_ would'
        print '                    ensure all form uploads have that path'
        print '                    prepended to the browser-given file name.'
        print '          The URL to redirect the browser to after'
        print '                    the uploads have completed.'
        print '     The maximum file size per file uploaded.'
        print '    The maximum number of uploaded files'
        print '                    allowed.'
        print '           The number of seconds from now to allow'
        print '                    the form post to begin.'
        print '               The X-Account-Meta-Temp-URL-Key for the'
        print '                    account.'
        print
        print 'Example output:'
        print '    Expires: 1323842228'
        print '  Signature: 18de97e47345a82c4dbfb3b06a640dbb'
        exit(1)
    path, redirect, max_file_size, max_file_count, seconds, key = argv[1:]
    try:
        max_file_size = int(max_file_size)
    except ValueError:
        max_file_size = -1
    if max_file_size < 0:
        print 'Please use a  value greater than or equal to 0.'
        exit(1)
    try:
        max_file_count = int(max_file_count)
    except ValueError:
        max_file_count = 0
    if max_file_count < 1:
        print 'Please use a positive  value.'
        exit(1)
    try:
        expires = int(time() + int(seconds))
    except ValueError:
        expires = 0
    if expires < 1:
        print 'Please use a positive  value.'
        exit(1)
    parts = path.split('/', 4)
    # Must be four parts, ['', 'v1', 'a', 'c'], must be a v1 request, have
    # account and container values, and optionally have an object prefix.
    if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \
            not parts[3]:
        print ' must point to a container at least.'
        print 'For example: /v1/account/container'
        print '         Or: /v1/account/container/object_prefix'
        exit(1)
    sig = hmac.new(key, '%s\n%s\n%s\n%s\n%s' % (path, redirect, max_file_size,
                                                max_file_count, expires),
                   sha1).hexdigest()
    print '  Expires:', expires
    print 'Signature:', sig
swift-1.13.1/bin/swift-object-updater0000775000175400017540000000155712323703611020635 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.obj.updater import ObjectUpdater
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(ObjectUpdater, conf_file, **options)
swift-1.13.1/bin/swift-recon-cron0000775000175400017540000000503512323703611017765 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
swift-recon-cron.py
"""

import os
import sys
from ConfigParser import ConfigParser
from gettext import gettext as _

from swift.common.utils import get_logger, dump_recon_cache


def get_async_count(device_dir, logger):
    async_count = 0
    for i in os.listdir(device_dir):
        asyncdir = os.path.join(device_dir, i, "async_pending")
        if os.path.isdir(asyncdir):
            for entry in os.listdir(asyncdir):
                if os.path.isdir(os.path.join(asyncdir, entry)):
                    async_hdir = os.path.join(asyncdir, entry)
                    async_count += len(os.listdir(async_hdir))
    return async_count


def main():
    c = ConfigParser()
    try:
        conf_path = sys.argv[1]
    except Exception:
        print "Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1]
        print "ex: swift-recon-cron /etc/swift/object-server.conf"
        sys.exit(1)
    if not c.read(conf_path):
        print "Unable to read config file %s" % conf_path
        sys.exit(1)
    conf = dict(c.items('filter:recon'))
    device_dir = conf.get('devices', '/srv/node')
    recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift')
    recon_lock_path = conf.get('recon_lock_path', '/var/lock')
    cache_file = os.path.join(recon_cache_path, "object.recon")
    lock_dir = os.path.join(recon_lock_path, "swift-recon-object-cron")
    conf['log_name'] = conf.get('log_name', 'recon-cron')
    logger = get_logger(conf, log_route='recon-cron')
    try:
        os.mkdir(lock_dir)
    except OSError as e:
        logger.critical(str(e))
        print str(e)
        sys.exit(1)
    try:
        asyncs = get_async_count(device_dir, logger)
        dump_recon_cache({'async_pending': asyncs}, cache_file, logger)
    except Exception:
        logger.exception(
            _('Exception during recon-cron while accessing devices'))
    try:
        os.rmdir(lock_dir)
    except Exception:
        logger.exception(_('Exception remove cronjob lock'))

if __name__ == '__main__':
    main()
swift-1.13.1/bin/swift-dispersion-report0000775000175400017540000003703612323703611021416 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
from ConfigParser import ConfigParser
from optparse import OptionParser
from sys import exit, stdout, stderr
from time import time
try:
    import simplejson as json
except ImportError:
    import json

from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool

from swift.common import direct_client
try:
    from swiftclient import get_auth
except ImportError:
    from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.exceptions import ClientException
from swift.common.utils import compute_eta, get_time_units, config_true_value


unmounted = []
notfound = []
json_output = False
debug = False
insecure = False


def get_error_log(prefix):

    def error_log(msg_or_exc):
        global debug, unmounted, notfound
        if hasattr(msg_or_exc, 'http_status'):
            identifier = '%s:%s/%s' % (msg_or_exc.http_host,
                                       msg_or_exc.http_port,
                                       msg_or_exc.http_device)
            if msg_or_exc.http_status == 507:
                if identifier not in unmounted:
                    unmounted.append(identifier)
                    print >>stderr, 'ERROR: %s is unmounted -- This will ' \
                        'cause replicas designated for that device to be ' \
                        'considered missing until resolved or the ring is ' \
                        'updated.' % (identifier)
                    stderr.flush()
            if debug and identifier not in notfound:
                notfound.append(identifier)
                print >>stderr, 'ERROR: %s returned a 404' % (identifier)
                stderr.flush()
        if not hasattr(msg_or_exc, 'http_status') or \
                msg_or_exc.http_status not in (404, 507):
            print >>stderr, 'ERROR: %s: %s' % (prefix, msg_or_exc)
            stderr.flush()
    return error_log


def container_dispersion_report(coropool, connpool, account, container_ring,
                                retries, output_missing_partitions):
    with connpool.item() as conn:
        containers = [c['name'] for c in conn.get_account(
            prefix='dispersion_', full_listing=True)[1]]
    containers_listed = len(containers)
    if not containers_listed:
        print >>stderr, 'No containers to query. Has ' \
                        'swift-dispersion-populate been run?'
        stderr.flush()
        return
    retries_done = [0]
    containers_queried = [0]
    container_copies_missing = defaultdict(int)
    container_copies_found = [0]
    container_copies_expected = [0]
    begun = time()
    next_report = [time() + 2]

    def direct(container, part, nodes):
        found_count = 0
        for node in nodes:
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, _junk = direct_client.retry(
                    direct_client.direct_head_container, node, part, account,
                    container, error_log=error_log, retries=retries)
                retries_done[0] += attempts - 1
                found_count += 1
            except ClientException as err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                              container, err))
            except (Exception, Timeout) as err:
                error_log('Giving up on /%s/%s/%s: %s' % (part, account,
                          container, err))
        if output_missing_partitions and \
                found_count < len(nodes):
            missing = len(nodes) - found_count
            print '\r\x1B[K',
            stdout.flush()
            print >>stderr, '# Container partition %s missing %s cop%s' % (
                part, missing, 'y' if missing == 1 else 'ies')
        container_copies_found[0] += found_count
        containers_queried[0] += 1
        container_copies_missing[len(nodes) - found_count] += 1
        if time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, containers_queried[0],
                                        containers_listed)
            if not json_output:
                print '\r\x1B[KQuerying containers: %d of %d, %d%s left, %d ' \
                      'retries' % (containers_queried[0], containers_listed,
                      round(eta), eta_unit, retries_done[0]),
                stdout.flush()
    container_parts = {}
    for container in containers:
        part, nodes = container_ring.get_nodes(account, container)
        if part not in container_parts:
            container_copies_expected[0] += len(nodes)
            container_parts[part] = part
            coropool.spawn(direct, container, part, nodes)
    coropool.waitall()
    distinct_partitions = len(container_parts)
    copies_found = container_copies_found[0]
    copies_expected = container_copies_expected[0]
    value = 100.0 * copies_found / copies_expected
    elapsed, elapsed_unit = get_time_units(time() - begun)
    container_copies_missing.pop(0, None)
    if not json_output:
        print '\r\x1B[KQueried %d containers for dispersion reporting, ' \
              '%d%s, %d retries' % (containers_listed, round(elapsed),
              elapsed_unit, retries_done[0])
        if containers_listed - distinct_partitions:
            print 'There were %d overlapping partitions' % (
                  containers_listed - distinct_partitions)
        for missing_copies, num_parts in container_copies_missing.iteritems():
            print missing_string(num_parts, missing_copies,
                                 container_ring.replica_count)
        print '%.02f%% of container copies found (%d of %d)' % (
            value, copies_found, copies_expected)
        print 'Sample represents %.02f%% of the container partition space' % (
            100.0 * distinct_partitions / container_ring.partition_count)
        stdout.flush()
        return None
    else:
        results = {'retries': retries_done[0],
                   'overlapping': containers_listed - distinct_partitions,
                   'pct_found': value,
                   'copies_found': copies_found,
                   'copies_expected': copies_expected}
        for missing_copies, num_parts in container_copies_missing.iteritems():
            results['missing_%d' % (missing_copies)] = num_parts
        return results


def object_dispersion_report(coropool, connpool, account, object_ring,
                             retries, output_missing_partitions):
    container = 'dispersion_objects'
    with connpool.item() as conn:
        try:
            objects = [o['name'] for o in conn.get_container(
                container, prefix='dispersion_', full_listing=True)[1]]
        except ClientException as err:
            if err.http_status != 404:
                raise
            print >>stderr, 'No objects to query. Has ' \
                            'swift-dispersion-populate been run?'
            stderr.flush()
            return
    objects_listed = len(objects)
    if not objects_listed:
        print >>stderr, 'No objects to query. Has swift-dispersion-populate ' \
                        'been run?'
        stderr.flush()
        return
    retries_done = [0]
    objects_queried = [0]
    object_copies_found = [0]
    object_copies_expected = [0]
    object_copies_missing = defaultdict(int)
    begun = time()
    next_report = [time() + 2]

    def direct(obj, part, nodes):
        found_count = 0
        for node in nodes:
            error_log = get_error_log('%(ip)s:%(port)s/%(device)s' % node)
            try:
                attempts, _junk = direct_client.retry(
                    direct_client.direct_head_object, node, part, account,
                    container, obj, error_log=error_log, retries=retries)
                retries_done[0] += attempts - 1
                found_count += 1
            except ClientException as err:
                if err.http_status not in (404, 507):
                    error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
                              container, obj, err))
            except (Exception, Timeout) as err:
                error_log('Giving up on /%s/%s/%s/%s: %s' % (part, account,
                          container, obj, err))
        if output_missing_partitions and \
                found_count < len(nodes):
            missing = len(nodes) - found_count
            print '\r\x1B[K',
            stdout.flush()
            print >>stderr, '# Object partition %s missing %s cop%s' % (
                part, missing, 'y' if missing == 1 else 'ies')
        object_copies_found[0] += found_count
        object_copies_missing[len(nodes) - found_count] += 1
        objects_queried[0] += 1
        if time() >= next_report[0]:
            next_report[0] = time() + 5
            eta, eta_unit = compute_eta(begun, objects_queried[0],
                                        objects_listed)
            if not json_output:
                print '\r\x1B[KQuerying objects: %d of %d, %d%s left, %d ' \
                      'retries' % (objects_queried[0], objects_listed,
                                   round(eta), eta_unit, retries_done[0]),
            stdout.flush()
    object_parts = {}
    for obj in objects:
        part, nodes = object_ring.get_nodes(account, container, obj)
        if part not in object_parts:
            object_copies_expected[0] += len(nodes)
            object_parts[part] = part
            coropool.spawn(direct, obj, part, nodes)
    coropool.waitall()
    distinct_partitions = len(object_parts)
    copies_found = object_copies_found[0]
    copies_expected = object_copies_expected[0]
    value = 100.0 * copies_found / copies_expected
    elapsed, elapsed_unit = get_time_units(time() - begun)
    if not json_output:
        print '\r\x1B[KQueried %d objects for dispersion reporting, ' \
              '%d%s, %d retries' % (objects_listed, round(elapsed),
              elapsed_unit, retries_done[0])
        if objects_listed - distinct_partitions:
            print 'There were %d overlapping partitions' % (
                  objects_listed - distinct_partitions)

        for missing_copies, num_parts in object_copies_missing.iteritems():
            print missing_string(num_parts, missing_copies,
                                 object_ring.replica_count)

        print '%.02f%% of object copies found (%d of %d)' % \
            (value, copies_found, copies_expected)
        print 'Sample represents %.02f%% of the object partition space' % (
            100.0 * distinct_partitions / object_ring.partition_count)
        stdout.flush()
        return None
    else:
        results = {'retries': retries_done[0],
                   'overlapping': objects_listed - distinct_partitions,
                   'pct_found': value,
                   'copies_found': copies_found,
                   'copies_expected': copies_expected}

        for missing_copies, num_parts in object_copies_missing.iteritems():
            results['missing_%d' % (missing_copies,)] = num_parts
        return results


def missing_string(partition_count, missing_copies, copy_count):
    exclamations = ''
    missing_string = str(missing_copies)
    if missing_copies == copy_count:
        exclamations = '!!! '
        missing_string = 'all'
    elif copy_count - missing_copies == 1:
        exclamations = '! '

    verb_string = 'was'
    partition_string = 'partition'
    if partition_count > 1:
        verb_string = 'were'
        partition_string = 'partitions'

    copy_string = 'copy'
    if missing_copies > 1:
        copy_string = 'copies'

    return '%sThere %s %d %s missing %s %s.' % (
        exclamations, verb_string, partition_count, partition_string,
        missing_string, copy_string
    )


if __name__ == '__main__':
    patcher.monkey_patch()
    hubs.get_hub().debug_exceptions = False

    conffile = '/etc/swift/dispersion.conf'

    parser = OptionParser(usage='''
Usage: %%prog [options] [conf_file]

[conf_file] defaults to %s'''.strip() % conffile)
    parser.add_option('-j', '--dump-json', action='store_true', default=False,
                      help='dump dispersion report in json format')
    parser.add_option('-d', '--debug', action='store_true', default=False,
                      help='print 404s to standard error')
    parser.add_option('-p', '--partitions', action='store_true', default=False,
                      help='print missing partitions to standard error')
    parser.add_option('--container-only', action='store_true', default=False,
                      help='Only run container report')
    parser.add_option('--object-only', action='store_true', default=False,
                      help='Only run object report')
    parser.add_option('--insecure', action='store_true', default=False,
                      help='Allow accessing insecure keystone server. '
                           'The keystone\'s certificate will not be verified.')
    options, args = parser.parse_args()

    if args:
        conffile = args.pop(0)

    c = ConfigParser()
    if not c.read(conffile):
        exit('Unable to read config file: %s' % conffile)
    conf = dict(c.items('dispersion'))
    swift_dir = conf.get('swift_dir', '/etc/swift')
    retries = int(conf.get('retries', 5))
    concurrency = int(conf.get('concurrency', 25))
    endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
    if options.dump_json or config_true_value(conf.get('dump_json', 'no')):
        json_output = True
    container_report = config_true_value(conf.get('container_report', 'yes')) \
        and not options.object_only
    object_report = config_true_value(conf.get('object_report', 'yes')) \
        and not options.container_only
    if not (object_report or container_report):
        exit("Neither container or object report is set to run")
    insecure = options.insecure \
        or config_true_value(conf.get('keystone_api_insecure', 'no'))
    if options.debug:
        debug = True

    coropool = GreenPool(size=concurrency)

    os_options = {'endpoint_type': endpoint_type}

    url, token = get_auth(conf['auth_url'], conf['auth_user'],
                          conf['auth_key'],
                          auth_version=conf.get('auth_version', '1.0'),
                          os_options=os_options,
                          insecure=insecure)
    account = url.rsplit('/', 1)[1]
    connpool = Pool(max_size=concurrency)
    connpool.create = lambda: SimpleClient(
        url=url, token=token, retries=retries)

    container_ring = Ring(swift_dir, ring_name='container')
    object_ring = Ring(swift_dir, ring_name='object')

    output = {}
    if container_report:
        output['container'] = container_dispersion_report(
            coropool, connpool, account, container_ring, retries,
            options.partitions)
    if object_report:
        output['object'] = object_dispersion_report(
            coropool, connpool, account, object_ring, retries,
            options.partitions)
    if json_output:
        print json.dumps(output)
swift-1.13.1/bin/swift-container-info0000775000175400017540000000210612323703611020627 0ustar  jenkinsjenkins00000000000000#!/usr/bin/python
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import sys
from optparse import OptionParser

from swift.cli.info import print_info, InfoSystemExit


if __name__ == '__main__':
    parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
    parser.add_option(
        '-d', '--swift-dir', default='/etc/swift',
        help="Pass location of swift directory")

    options, args = parser.parse_args()

    if len(args) != 1:
        sys.exit(parser.print_help())

    try:
        print_info('container', *args, **vars(options))
    except InfoSystemExit:
        sys.exit(1)
swift-1.13.1/bin/swift-proxy-server0000775000175400017540000000153512323703611020406 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi

if __name__ == '__main__':
    conf_file, options = parse_options()
    sys.exit(run_wsgi(conf_file, 'proxy-server', default_port=8080, **options))
swift-1.13.1/bin/swift-get-nodes0000775000175400017540000001157412323703611017612 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import optparse
import sys
import urllib

from swift.common.ring import Ring
from swift.common.utils import hash_path, storage_directory


parser = optparse.OptionParser()
parser.add_option('-a', '--all', action='store_true',
                  help='Show all handoff nodes')
parser.add_option('-p', '--partition', metavar='PARTITION',
                  help='Show nodes for a given partition')
(options, args) = parser.parse_args()

if (len(args) < 2 or len(args) > 4) and \
   (options.partition is None or not args):
    print 'Usage: %s [-a]   [] []' \
        % sys.argv[0]
    print '   Or: %s [-a]  -p partition' % sys.argv[0]
    print '   Note: account, container, object can also be a single arg ' \
        'separated by /'
    print 'Shows the nodes responsible for the item specified.'
    print 'Example:'
    print '    $ %s /etc/swift/account.ring.gz MyAccount' % sys.argv[0]
    print '    Partition 5743883'
    print '    Hash 96ae332a60b58910784e4417a03e1ad0'
    print '    10.1.1.7:8000 sdd1'
    print '    10.1.9.2:8000 sdb1'
    print '    10.1.5.5:8000 sdf1'
    print '    10.1.5.9:8000 sdt1 # [Handoff]'
    sys.exit(1)

if len(args) == 2 and '/' in args[1]:
    # Parse single path arg, as noted in above help text.
    path = args[1].lstrip('/')
    args = [args[0]] + [p for p in path.split('/', 2) if p]

ringloc = None
account = None
container = None
obj = None

if len(args) == 4:
    # Account, Container and Object
    ring_file, account, container, obj = args
    ring = Ring(ring_file)
    hash_str = hash_path(account, container, obj)
    part, nodes = ring.get_nodes(account, container, obj)
    target = "%s/%s/%s" % (account, container, obj)
    loc = 'objects'
elif len(args) == 3:
    # Account, Container
    ring_file, account, container = args
    ring = Ring(ring_file)
    hash_str = hash_path(account, container)
    part, nodes = ring.get_nodes(account, container)
    target = "%s/%s" % (account, container)
    loc = 'containers'
elif len(args) == 2:
    # Account
    ring_file, account = args
    ring = Ring(ring_file)
    hash_str = hash_path(account)
    part, nodes = ring.get_nodes(account)
    target = "%s" % (account)
    loc = 'accounts'
elif len(args) == 1:
    # Partition
    ring_file = args[0]
    ring = Ring(ring_file)
    hash_str = None
    part = int(options.partition)
    nodes = ring.get_part_nodes(part)
    target = ''
    loc = ring_file.rsplit('/', 1)[-1].split('.', 1)[0]
    if loc in ('account', 'container', 'object'):
        loc += 's'
    else:
        loc = ''

more_nodes = []
for more_node in ring.get_more_nodes(part):
    more_nodes.append(more_node)
    if not options.all and len(more_nodes) >= len(nodes):
        break

print '\nAccount \t%s' % account
print 'Container\t%s' % container
print 'Object   \t%s\n' % obj
print '\nPartition\t%s' % part
print 'Hash       \t%s\n' % hash_str

for node in nodes:
    print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
                                            node['device'])
for mnode in more_nodes:
    print 'Server:Port Device\t%s:%s %s\t [Handoff]' \
        % (mnode['ip'], mnode['port'], mnode['device'])
print "\n"
for node in nodes:
    print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
        % (node['ip'], node['port'], node['device'], part,
           urllib.quote(target))
for mnode in more_nodes:
    print 'curl -I -XHEAD "http://%s:%s/%s/%s/%s" # [Handoff]' \
        % (mnode['ip'], mnode['port'], mnode['device'], part,
           urllib.quote(target))
print "\n"
print 'Use your own device location of servers:'
print 'such as "export DEVICE=/srv/node"'
for node in nodes:
    if hash_str:
        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/"' % (
            node['ip'], node['device'], storage_directory(loc, part, hash_str))
    else:
        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/"' % (
            node['ip'], node['device'], loc, part)
for mnode in more_nodes:
    if hash_str:
        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/" '\
              '# [Handoff]' % (mnode['ip'], mnode['device'],
                               storage_directory(loc, part, hash_str))
    else:
        print 'ssh %s "ls -lah ${DEVICE:-/srv/node}/%s/%s/%s/" # [Handoff]' % (
            mnode['ip'], mnode['device'], loc, part)
swift-1.13.1/bin/swift-init0000775000175400017540000000615512323703611016667 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from optparse import OptionParser

from swift.common.manager import Manager, UnknownCommandError, \
    KILL_WAIT, RUN_DIR

USAGE = """%prog  [ ...]  [options]

Commands:
""" + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])


def main():
    parser = OptionParser(USAGE)
    parser.add_option('-v', '--verbose', action="store_true",
                      default=False, help="display verbose output")
    parser.add_option('-w', '--no-wait', action="store_false", dest="wait",
                      default=True, help="won't wait for server to start "
                      "before returning")
    parser.add_option('-o', '--once', action="store_true",
                      default=False, help="only run one pass of daemon")
    # this is a negative option, default is options.daemon = True
    parser.add_option('-n', '--no-daemon', action="store_false", dest="daemon",
                      default=True, help="start server interactively")
    parser.add_option('-g', '--graceful', action="store_true",
                      default=False, help="send SIGHUP to supporting servers")
    parser.add_option('-c', '--config-num', metavar="N", type="int",
                      dest="number", default=0,
                      help="send command to the Nth server only")
    parser.add_option('-k', '--kill-wait', metavar="N", type="int",
                      dest="kill_wait", default=KILL_WAIT,
                      help="wait N seconds for processes to die (default 15)")
    parser.add_option('-r', '--run-dir', type="str",
                      dest="run_dir", default=RUN_DIR,
                      help="alternative directory to store running pid files "
                      "default: %s" % RUN_DIR)

    options, args = parser.parse_args()

    if len(args) < 2:
        parser.print_help()
        print 'ERROR: specify server(s) and command'
        return 1

    command = args[-1]
    servers = args[:-1]

    # this is just a silly swap for me cause I always try to "start main"
    commands = dict(Manager.list_commands()).keys()
    if command not in commands and servers[0] in commands:
        servers.append(command)
        command = servers.pop(0)

    manager = Manager(servers, run_dir=options.run_dir)
    try:
        status = manager.run_command(command, **options.__dict__)
    except UnknownCommandError:
        parser.print_help()
        print 'ERROR: unknown command, %s' % command
        status = 1

    return 1 if status else 0


if __name__ == "__main__":
    sys.exit(main())
swift-1.13.1/bin/swift-container-sync0000775000175400017540000000155612323703611020660 0ustar  jenkinsjenkins00000000000000#!/usr/bin/python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.container.sync import ContainerSync
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(ContainerSync, conf_file, **options)
swift-1.13.1/bin/swift-container-replicator0000775000175400017540000000160412323703611022042 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.container.replicator import ContainerReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(ContainerReplicator, conf_file, **options)
swift-1.13.1/bin/swift-temp-url0000775000175400017540000000546712323703611017476 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hmac
from hashlib import sha1
from os.path import basename
from sys import argv, exit, stderr
from time import time


if __name__ == '__main__':
    if len(argv) != 5:
        prog = basename(argv[0])
        print 'Syntax: %s    ' % prog
        print
        print 'Where:'
        print '     The method to allow; GET for example.'
        print '    The number of seconds from now to allow requests.'
        print '       The full path to the resource.'
        print '             Example: /v1/AUTH_account/c/o'
        print '        The X-Account-Meta-Temp-URL-Key for the account.'
        print
        print 'Example output:'
        print '    /v1/AUTH_account/c/o?temp_url_sig=34d49efc32fe6e3082e411e' \
              'eeb85bd8a&temp_url_expires=1323482948'
        print
        print 'This can be used to form a URL to give out for the access '
        print 'allowed. For example:'
        print '    echo https://swift-cluster.example.com`%s GET 60 ' \
              '/v1/AUTH_account/c/o mykey`' % prog
        print
        print 'Might output:'
        print '    https://swift-cluster.example.com/v1/AUTH_account/c/o?' \
              'temp_url_sig=34d49efc32fe6e3082e411eeeb85bd8a&' \
              'temp_url_expires=1323482948'
        exit(1)
    method, seconds, path, key = argv[1:]
    try:
        expires = int(time() + int(seconds))
    except ValueError:
        expires = 0
    if expires < 1:
        print 'Please use a positive  value.'
        exit(1)
    parts = path.split('/', 4)
    # Must be five parts, ['', 'v1', 'a', 'c', 'o'], must be a v1 request, have
    # account, container, and object values, and the object value can't just
    # have '/'s.
    if len(parts) != 5 or parts[0] or parts[1] != 'v1' or not parts[2] or \
            not parts[3] or not parts[4].strip('/'):
        stderr.write(
            'WARNING: "%s" does not refer to an object '
            '(e.g. /v1/account/container/object).\n' % path)
        stderr.write(
            'WARNING: Non-object paths will be rejected by tempurl.\n')
    sig = hmac.new(key, '%s\n%s\n%s' % (method, expires, path),
                   sha1).hexdigest()
    print '%s?temp_url_sig=%s&temp_url_expires=%s' % (path, sig, expires)
swift-1.13.1/bin/swift-object-server0000775000175400017540000000173212323703611020472 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi
from swift.obj import server


if __name__ == '__main__':
    conf_file, options = parse_options()
    sys.exit(run_wsgi(conf_file, 'object-server', default_port=6000,
                      global_conf_callback=server.global_conf_callback,
                      **options))
swift-1.13.1/bin/swift-account-info0000775000175400017540000000210212323703611020275 0ustar  jenkinsjenkins00000000000000#!/usr/bin/python
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import sys
from optparse import OptionParser

from swift.cli.info import print_info, InfoSystemExit


if __name__ == '__main__':
    parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
    parser.add_option(
        '-d', '--swift-dir', default='/etc/swift',
        help="Pass location of swift directory")

    options, args = parser.parse_args()

    if len(args) != 1:
        sys.exit(parser.print_help())

    try:
        print_info('account', *args, **vars(options))
    except InfoSystemExit:
        sys.exit(1)
swift-1.13.1/bin/swift-object-expirer0000775000175400017540000000274312323703611020645 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.common.daemon import run_daemon
from swift.common.utils import parse_options
from swift.obj.expirer import ObjectExpirer
from optparse import OptionParser

if __name__ == '__main__':
    parser = OptionParser("%prog CONFIG [options]")
    parser.add_option('--processes', dest='processes',
                      help="Number of processes to use to do the work, don't "
                      "use this option to do all the work in one process")
    parser.add_option('--process', dest='process',
                      help="Process number for this process, don't use "
                      "this option to do all the work in one process, this "
                      "is used to determine which part of the work this "
                      "process should do")
    conf_file, options = parse_options(parser=parser, once=True)
    run_daemon(ObjectExpirer, conf_file, **options)
swift-1.13.1/bin/swift-container-auditor0000775000175400017540000000157312323703611021352 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.container.auditor import ContainerAuditor
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(ContainerAuditor, conf_file, **options)
swift-1.13.1/bin/swift-account-reaper0000775000175400017540000000156212323703611020631 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.account.reaper import AccountReaper
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(AccountReaper, conf_file, **options)
swift-1.13.1/bin/swift-config0000775000175400017540000000574712323703611017177 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import optparse
import os
import sys

from swift.common.manager import Server
from swift.common.utils import readconf
from swift.common.wsgi import appconfig

parser = optparse.OptionParser('%prog [options] SERVER')
parser.add_option('-c', '--config-num', metavar="N", type="int",
                  dest="number", default=0,
                  help="parse config for the Nth server only")
parser.add_option('-s', '--section', help="only display matching sections")
parser.add_option('-w', '--wsgi', action='store_true',
                  help="use wsgi/paste parser instead of readconf")


def _context_name(context):
    return ':'.join((context.object_type.name, context.name))


def inspect_app_config(app_config):
    conf = {}
    context = app_config.context
    section_name = _context_name(context)
    conf[section_name] = context.config()
    if context.object_type.name == 'pipeline':
        filters = context.filter_contexts
        pipeline = []
        for filter_context in filters:
            conf[_context_name(filter_context)] = filter_context.config()
            pipeline.append(filter_context.entry_point_name)
        app_context = context.app_context
        conf[_context_name(app_context)] = app_context.config()
        pipeline.append(app_context.entry_point_name)
        conf[section_name]['pipeline'] = ' '.join(pipeline)
    return conf


def main():
    options, args = parser.parse_args()
    options = dict(vars(options))

    if not args:
        return 'ERROR: specify type of server or conf_path'
    conf_files = []
    for arg in args:
        if os.path.exists(arg):
            conf_files.append(arg)
        else:
            conf_files += Server(arg).conf_files(**options)
    for conf_file in conf_files:
        print '# %s' % conf_file
        if options['wsgi']:
            app_config = appconfig(conf_file)
            conf = inspect_app_config(app_config)
        else:
            conf = readconf(conf_file)
        flat_vars = {}
        for k, v in conf.items():
            if options['section'] and k != options['section']:
                continue
            if not isinstance(v, dict):
                flat_vars[k] = v
                continue
            print '[%s]' % k
            for opt, value in v.items():
                print '%s = %s' % (opt, value)
            print
        for k, v in flat_vars.items():
            print '# %s = %s' % (k, v)
        print

if __name__ == "__main__":
    sys.exit(main())
swift-1.13.1/bin/swift-account-replicator0000775000175400017540000000157612323703611021524 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from swift.account.replicator import AccountReplicator
from swift.common.utils import parse_options
from swift.common.daemon import run_daemon

if __name__ == '__main__':
    conf_file, options = parse_options(once=True)
    run_daemon(AccountReplicator, conf_file, **options)
swift-1.13.1/bin/swift-account-server0000775000175400017540000000156512323703611020664 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from swift.common.utils import parse_options
from swift.common.wsgi import run_wsgi

if __name__ == '__main__':
    conf_file, options = parse_options()
    sys.exit(run_wsgi(conf_file,
                      'account-server', default_port=6002, **options))
swift-1.13.1/bin/swift-dispersion-populate0000775000175400017540000001701112323703611021723 0ustar  jenkinsjenkins00000000000000#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import traceback
from ConfigParser import ConfigParser
from cStringIO import StringIO
from optparse import OptionParser
from sys import exit, stdout
from time import time

from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool

try:
    from swiftclient import get_auth
except ImportError:
    from swift.common.internal_client import get_auth
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units, config_true_value


insecure = False


def put_container(connpool, container, report):
    global retries_done
    try:
        with connpool.item() as conn:
            conn.put_container(container)
            retries_done += conn.attempts - 1
        if report:
            report(True)
    except Exception:
        if report:
            report(False)
        raise


def put_object(connpool, container, obj, report):
    global retries_done
    try:
        with connpool.item() as conn:
            conn.put_object(container, obj, StringIO(obj),
                            headers={'x-object-meta-dispersion': obj})
            retries_done += conn.attempts - 1
        if report:
            report(True)
    except Exception:
        if report:
            report(False)
        raise


def report(success):
    global begun, created, item_type, next_report, need_to_create, retries_done
    if not success:
        traceback.print_exc()
        exit('Gave up due to error(s).')
    created += 1
    if time() < next_report:
        return
    next_report = time() + 5
    eta, eta_unit = compute_eta(begun, created, need_to_create)
    print '\r\x1B[KCreating %s: %d of %d, %d%s left, %d retries' % (item_type,
          created, need_to_create, round(eta), eta_unit, retries_done),
    stdout.flush()


if __name__ == '__main__':
    global begun, created, item_type, next_report, need_to_create, retries_done
    patcher.monkey_patch()

    conffile = '/etc/swift/dispersion.conf'

    parser = OptionParser(usage='''
Usage: %%prog [options] [conf_file]

[conf_file] defaults to %s'''.strip() % conffile)
    parser.add_option('--container-only', action='store_true', default=False,
                      help='Only run container population')
    parser.add_option('--object-only', action='store_true', default=False,
                      help='Only run object population')
    parser.add_option('--container-suffix-start', type=int, default=0,
                      help='container suffix start value, defaults to 0')
    parser.add_option('--object-suffix-start', type=int, default=0,
                      help='object suffix start value, defaults to 0')
    parser.add_option('--insecure', action='store_true', default=False,
                      help='Allow accessing insecure keystone server. '
                           'The keystone\'s certificate will not be verified.')
    options, args = parser.parse_args()

    if args:
        conffile = args.pop(0)

    c = ConfigParser()
    if not c.read(conffile):
        exit('Unable to read config file: %s' % conffile)
    conf = dict(c.items('dispersion'))
    swift_dir = conf.get('swift_dir', '/etc/swift')
    dispersion_coverage = float(conf.get('dispersion_coverage', 1))
    retries = int(conf.get('retries', 5))
    concurrency = int(conf.get('concurrency', 25))
    endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
    insecure = options.insecure \
        or config_true_value(conf.get('keystone_api_insecure', 'no'))
    container_populate = config_true_value(
        conf.get('container_populate', 'yes')) and not options.object_only
    object_populate = config_true_value(
        conf.get('object_populate', 'yes')) and not options.container_only

    if not (object_populate or container_populate):
        exit("Neither container or object populate is set to run")

    coropool = GreenPool(size=concurrency)
    retries_done = 0

    os_options = {'endpoint_type': endpoint_type}

    url, token = get_auth(conf['auth_url'], conf['auth_user'],
                          conf['auth_key'],
                          auth_version=conf.get('auth_version', '1.0'),
                          os_options=os_options,
                          insecure=insecure)
    account = url.rsplit('/', 1)[1]
    connpool = Pool(max_size=concurrency)
    connpool.create = lambda: SimpleClient(
        url=url, token=token, retries=retries)

    if container_populate:
        container_ring = Ring(swift_dir, ring_name='container')
        parts_left = dict((x, x)
                          for x in xrange(container_ring.partition_count))
        item_type = 'containers'
        created = 0
        retries_done = 0
        need_to_create = need_to_queue = \
            dispersion_coverage / 100.0 * container_ring.partition_count
        begun = next_report = time()
        next_report += 2
        suffix = 0
        while need_to_queue >= 1:
            container = 'dispersion_%d' % suffix
            part, _junk = container_ring.get_nodes(account, container)
            if part in parts_left:
                if suffix >= options.container_suffix_start:
                    coropool.spawn(put_container, connpool, container, report)
                    sleep()
                else:
                    report(True)
                del parts_left[part]
                need_to_queue -= 1
            suffix += 1
        coropool.waitall()
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KCreated %d containers for dispersion reporting, ' \
              '%d%s, %d retries' % \
              (need_to_create, round(elapsed), elapsed_unit, retries_done)
        stdout.flush()

    if object_populate:
        container = 'dispersion_objects'
        put_container(connpool, container, None)
        object_ring = Ring(swift_dir, ring_name='object')
        parts_left = dict((x, x) for x in xrange(object_ring.partition_count))
        item_type = 'objects'
        created = 0
        retries_done = 0
        need_to_create = need_to_queue = \
            dispersion_coverage / 100.0 * object_ring.partition_count
        begun = next_report = time()
        next_report += 2
        suffix = 0
        while need_to_queue >= 1:
            obj = 'dispersion_%d' % suffix
            part, _junk = object_ring.get_nodes(account, container, obj)
            if part in parts_left:
                if suffix >= options.object_suffix_start:
                    coropool.spawn(
                        put_object, connpool, container, obj, report)
                    sleep()
                else:
                    report(True)
                del parts_left[part]
                need_to_queue -= 1
            suffix += 1
        coropool.waitall()
        elapsed, elapsed_unit = get_time_units(time() - begun)
        print '\r\x1B[KCreated %d objects for dispersion reporting, ' \
              '%d%s, %d retries' % \
              (need_to_create, round(elapsed), elapsed_unit, retries_done)
        stdout.flush()
swift-1.13.1/etc/0000775000175400017540000000000012323703665014651 5ustar  jenkinsjenkins00000000000000swift-1.13.1/etc/swift.conf-sample0000664000175400017540000000465112323703614020133 0ustar  jenkinsjenkins00000000000000[swift-hash]

# swift_hash_path_suffix and swift_hash_path_prefix are used as part of the
# the hashing algorithm when determining data placement in the cluster.
# These values should remain secret and MUST NOT change
# once a cluster has been deployed.

swift_hash_path_suffix = changeme
swift_hash_path_prefix = changeme


# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster.

[swift-constraints]

# max_file_size is the largest "normal" object that can be saved in
# the cluster. This is also the limit on the size of each segment of
# a "large" object when using the large object manifest support.
# This value is set in bytes. Setting it to lower than 1MiB will cause
# some tests to fail. It is STRONGLY recommended to leave this value at
# the default (5 * 2**30 + 2).

#max_file_size = 5368709122


# max_meta_name_length is the max number of bytes in the utf8 encoding
# of the name portion of a metadata header.

#max_meta_name_length = 128


# max_meta_value_length is the max number of bytes in the utf8 encoding
# of a metadata value

#max_meta_value_length = 256


# max_meta_count is the max number of metadata keys that can be stored
# on a single account, container, or object

#max_meta_count = 90


# max_meta_overall_size is the max number of bytes in the utf8 encoding
# of the metadata (keys + values)

#max_meta_overall_size = 4096

# max_header_size is the max number of bytes in the utf8 encoding of each
# header. Using 8192 as default because eventlet use 8192 as max size of
# header line. This value may need to be increased when using identity
# v3 API tokens including more than 7 catalog entries.
# See also include_service_catalog in proxy-server.conf-sample
# (documented in overview_auth.rst)

#max_header_size = 8192


# max_object_name_length is the max number of bytes in the utf8 encoding
# of an object name

#max_object_name_length = 1024


# container_listing_limit is the default (and max) number of items
# returned for a container listing request

#container_listing_limit = 10000


# account_listing_limit is the default (and max) number of items returned
# for an account listing request
#account_listing_limit = 10000


# max_account_name_length is the max number of bytes in the utf8 encoding
# of an account name

#max_account_name_length = 256


# max_container_name_length is the max number of bytes in the utf8 encoding
# of a container name

#max_container_name_length = 256
swift-1.13.1/etc/swift-rsyslog.conf-sample0000664000175400017540000000403412323703611021623 0ustar  jenkinsjenkins00000000000000# Uncomment the following to have a log containing all logs together
#local.* /var/log/swift/all.log

# Uncomment the following to have hourly swift logs.
#$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%"
#local0.* ?HourlyProxyLog

# Use the following to have separate log files for each of the main servers:
# account-server, container-server, object-server, proxy-server. Note:
# object-updater's output will be stored in object.log.
if $programname contains 'swift' then /var/log/swift/swift.log
if $programname contains 'account' then /var/log/swift/account.log
if $programname contains 'container' then /var/log/swift/container.log
if $programname contains 'object' then /var/log/swift/object.log
if $programname contains 'proxy' then /var/log/swift/proxy.log

# Uncomment the following to have specific log via program name.
#if $programname == 'swift' then /var/log/swift/swift.log
#if $programname == 'account-server' then /var/log/swift/account-server.log
#if $programname == 'account-replicator' then /var/log/swift/account-replicator.log
#if $programname == 'account-auditor' then /var/log/swift/account-auditor.log
#if $programname == 'account-reaper' then /var/log/swift/account-reaper.log
#if $programname == 'container-server' then /var/log/swift/container-server.log
#if $programname == 'container-replicator' then /var/log/swift/container-replicator.log
#if $programname == 'container-updater' then /var/log/swift/container-updater.log
#if $programname == 'container-auditor' then /var/log/swift/container-auditor.log
#if $programname == 'container-sync' then /var/log/swift/container-sync.log
#if $programname == 'object-server' then /var/log/swift/object-server.log
#if $programname == 'object-replicator' then /var/log/swift/object-replicator.log
#if $programname == 'object-updater' then /var/log/swift/object-updater.log
#if $programname == 'object-auditor' then /var/log/swift/object-auditor.log

# Use the following to discard logs that don't match any of the above to avoid
# them filling up /var/log/messages.
local0.* ~
swift-1.13.1/etc/drive-audit.conf-sample0000664000175400017540000000125712323703611021210 0ustar  jenkinsjenkins00000000000000[drive-audit]
# device_dir = /srv/node
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
# minutes = 60
# error_limit = 1
#
# Location of the log file with globbing
# pattern to check against device errors.
# log_file_pattern = /var/log/kern*
#
# Regular expression patterns to be used to locate
# device blocks with errors in the log file. Currently
# the default ones are as follows:
#   \berror\b.*\b(sd[a-z]{1,2}\d?)\b
#   \b(sd[a-z]{1,2}\d?)\b.*\berror\b
# One can overwrite the default ones by providing
# new expressions using the format below: 
# Format: regex_pattern_X = regex_expression
# Example:
#   regex_pattern_1 = \berror\b.*\b(dm-[0-9]{1,2}\d?)\b
swift-1.13.1/etc/object-expirer.conf-sample0000664000175400017540000000346712323703611021722 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# swift_dir = /etc/swift
# user = swift
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =

[object-expirer]
# interval = 300
# auto_create_account_prefix = .
# expiring_objects_account_name = expiring_objects
# report_interval = 300
# concurrency is the level of concurrency o use to do the work, this value
# must be set to at least 1
# concurrency = 1
# processes is how many parts to divide the work into, one part per process
#   that will be doing the work
# processes set 0 means that a single process will be doing all the work
# processes can also be specified on the command line and will override the
#   config value
# processes = 0
# process is which of the parts a particular process will work on
# process can also be specified on the command line and will overide the config
#   value
# process is "zero based", if you want to use 3 processes, you should run
#  processes with process set to 0, 1, and 2
# process = 0

[pipeline:main]
pipeline = catch_errors cache proxy-server

[app:proxy-server]
use = egg:swift#proxy
# See proxy-server.conf-sample for options

[filter:cache]
use = egg:swift#memcache
# See proxy-server.conf-sample for options

[filter:catch_errors]
use = egg:swift#catch_errors
# See proxy-server.conf-sample for options
swift-1.13.1/etc/mime.types-sample0000664000175400017540000000052112323703611020132 0ustar  jenkinsjenkins00000000000000#########################################################
# A nice place to put custom Mime-Types for Swift       #
# Please enter Mime-Types in standard mime.types format #
# Mime-Type Extension ex. image/jpeg jpg                #
#########################################################

#EX. Mime-Type Extension
#    foo/bar   foo


swift-1.13.1/etc/proxy-server.conf-sample0000664000175400017540000005353412323703611021465 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 80
# bind_timeout = 30
# backlog = 4096
# swift_dir = /etc/swift
# user = swift

# Enables exposing configuration settings via HTTP GET /info.
# expose_info = true

# Key to use for admin calls that are HMAC signed.  Default is empty,
# which will disable admin calls to /info.
# admin_key = secret_admin_key
#
# Allows the ability to withhold sections from showing up in the public
# calls to /info.  The following would cause the sections 'container_quotas'
# and 'tempurl' to not be listed.  Default is empty, allowing all registered
# fetures to be listed via HTTP GET /info.
# disallowed_sections = container_quotas, tempurl

# Use an integer to override the number of pre-forked processes that will
# accept connections.  Should default to the number of effective cpu
# cores in the system.  It's worth noting that individual workers will
# use many eventlet co-routines to service multiple concurrent requests.
# workers = auto
#
# Maximum concurrent requests per worker
# max_clients = 1024
#
# Set the following two lines to enable SSL. This is for testing only.
# cert_file = /etc/swift/proxy.crt
# key_file = /etc/swift/proxy.key
#
# expiring_objects_container_divisor = 86400
# expiring_objects_account_name = expiring_objects
#
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_headers = false
# log_address = /dev/log
#
# This optional suffix (default is empty) that would be appended to the swift transaction
# id allows one to easily figure out from which cluster that X-Trans-Id belongs to.
# This is very useful when one is managing more than one swift cluster.
# trans_id_suffix =
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
#
# Use a comma separated list of full url (http://foo.bar:1234,https://foo.bar)
# cors_allow_origin =
#
# client_timeout = 60
# eventlet_debug = false

[pipeline:main]
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl slo dlo ratelimit tempauth container-quotas account-quotas proxy-logging proxy-server

[app:proxy-server]
use = egg:swift#proxy
# You can override the default log routing for this app here:
# set log_name = proxy-server
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_address = /dev/log
#
# log_handoffs = true
# recheck_account_existence = 60
# recheck_container_existence = 60
# object_chunk_size = 8192
# client_chunk_size = 8192
#
# How long the proxy server will wait on responses from the a/c/o servers.
# node_timeout = 10
#
# How long the proxy server will wait for an initial response and to read a
# chunk of data from the object servers while serving GET / HEAD requests.
# Timeouts from these requests can be recovered from so setting this to
# something lower than node_timeout would provide quicker error recovery
# while allowing for a longer timeout for non-recoverable requests (PUTs).
# Defaults to node_timeout, should be overriden if node_timeout is set to a
# high number to prevent client timeouts from firing before the proxy server
# has a chance to retry.
# recoverable_node_timeout = node_timeout
#
# conn_timeout = 0.5
#
# How long to wait for requests to finish after a quorum has been established.
# post_quorum_timeout = 0.5
#
# How long without an error before a node's error count is reset. This will
# also be how long before a node is reenabled after suppression is triggered.
# error_suppression_interval = 60
#
# How many errors can accumulate before a node is temporarily ignored.
# error_suppression_limit = 10
#
# If set to 'true' any authorized user may create and delete accounts; if
# 'false' no one, even authorized, can.
# allow_account_management = false
#
# Set object_post_as_copy = false to turn on fast posts where only the metadata
# changes are stored anew and the original data file is kept in place. This
# makes for quicker posts; but since the container metadata isn't updated in
# this mode, features like container sync won't be able to sync posts.
# object_post_as_copy = true
#
# If set to 'true' authorized accounts that do not yet exist within the Swift
# cluster will be automatically created.
# account_autocreate = false
#
# If set to a positive value, trying to create a container when the account
# already has at least this maximum containers will result in a 403 Forbidden.
# Note: This is a soft limit, meaning a user might exceed the cap for
# recheck_account_existence before the 403s kick in.
# max_containers_per_account = 0
#
# This is a comma separated list of account hashes that ignore the
# max_containers_per_account cap.
# max_containers_whitelist =
#
# Comma separated list of Host headers to which the proxy will deny requests.
# deny_host_headers =
#
# Prefix used when automatically creating accounts.
# auto_create_account_prefix = .
#
# Depth of the proxy put queue.
# put_queue_depth = 10
#
# Storage nodes can be chosen at random (shuffle), by using timing
# measurements (timing), or by using an explicit match (affinity).
# Using timing measurements may allow for lower overall latency, while
# using affinity allows for finer control. In both the timing and
# affinity cases, equally-sorting nodes are still randomly chosen to
# spread load.
# The valid values for sorting_method are "affinity", "shuffle", and "timing".
# sorting_method = shuffle
#
# If the "timing" sorting_method is used, the timings will only be valid for
# the number of seconds configured by timing_expiry.
# timing_expiry = 300
#
# The maximum time (seconds) that a large object connection is allowed to last.
# max_large_object_get_time = 86400
#
# Set to the number of nodes to contact for a normal request. You can use
# '* replicas' at the end to have it use the number given times the number of
# replicas for the ring being used for the request.
# request_node_count = 2 * replicas
#
# Which backend servers to prefer on reads. Format is r for region
# N or rz for region N, zone M. The value after the equals is
# the priority; lower numbers are higher priority.
#
# Example: first read from region 1 zone 1, then region 1 zone 2, then
# anything in region 2, then everything else:
# read_affinity = r1z1=100, r1z2=200, r2=300
# Default is empty, meaning no preference.
# read_affinity =
#
# Which backend servers to prefer on writes. Format is r for region
# N or rz for region N, zone M. If this is set, then when
# handling an object PUT request, some number (see setting
# write_affinity_node_count) of local backend servers will be tried
# before any nonlocal ones.
#
# Example: try to write to regions 1 and 2 before writing to any other
# nodes:
# write_affinity = r1, r2
# Default is empty, meaning no preference.
# write_affinity =
#
# The number of local (as governed by the write_affinity setting)
# nodes to attempt to contact first, before any non-local ones. You
# can use '* replicas' at the end to have it use the number given
# times the number of replicas for the ring being used for the
# request.
# write_affinity_node_count = 2 * replicas
#
# These are the headers whose values will only be shown to swift_owners. The
# exact definition of a swift_owner is up to the auth system in use, but
# usually indicates administrative responsibilities.
# swift_owner_headers = x-container-read, x-container-write, x-container-sync-key, x-container-sync-to, x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, x-account-access-control


[filter:tempauth]
use = egg:swift#tempauth
# You can override the default log routing for this filter here:
# set log_name = tempauth
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log
#
# The reseller prefix will verify a token begins with this prefix before even
# attempting to validate it. Also, with authorization, only Swift storage
# accounts with this prefix will be authorized by this middleware. Useful if
# multiple auth systems are in use for one Swift cluster.
# reseller_prefix = AUTH
#
# The auth prefix will cause requests beginning with this prefix to be routed
# to the auth subsystem, for granting tokens, etc.
# auth_prefix = /auth/
# token_life = 86400
#
# This allows middleware higher in the WSGI pipeline to override auth
# processing, useful for middleware such as tempurl and formpost. If you know
# you're not going to use such middleware and you want a bit of extra security,
# you can set this to false.
# allow_overrides = true
#
# This specifies what scheme to return with storage urls:
# http, https, or default (chooses based on what the server is running as)
# This can be useful with an SSL load balancer in front of a non-SSL server.
# storage_url_scheme = default
#
# Lastly, you need to list all the accounts/users you want here. The format is:
#   user__ =  [group] [group] [...] [storage_url]
# or if you want underscores in  or , you can base64 encode them
# (with no equal signs) and use this format:
#   user64__ =  [group] [group] [...] [storage_url]
# There are special groups of:
#   .reseller_admin = can do anything to any account for this auth
#   .admin = can do anything within the account
# If neither of these groups are specified, the user can only access containers
# that have been explicitly allowed for them by a .admin or .reseller_admin.
# The trailing optional storage_url allows you to specify an alternate url to
# hand back to the user upon authentication. If not specified, this defaults to
# $HOST/v1/_ where $HOST will do its best to resolve
# to what the requester would need to use to reach this host.
# Here are example entries, required for running the tests:
user_admin_admin = admin .admin .reseller_admin
user_test_tester = testing .admin
user_test2_tester2 = testing2 .admin
user_test_tester3 = testing3

# To enable Keystone authentication you need to have the auth token
# middleware first to be configured. Here is an example below, please
# refer to the keystone's documentation for details about the
# different settings.
#
# You'll need to have as well the keystoneauth middleware enabled
# and have it in your main pipeline so instead of having tempauth in
# there you can change it to: authtoken keystoneauth
#
# [filter:authtoken]
# paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory
# auth_host = keystonehost
# auth_port = 35357
# auth_protocol = http
# auth_uri = http://keystonehost:5000/
# admin_tenant_name = service
# admin_user = swift
# admin_password = password
# delay_auth_decision = 1
# cache = swift.cache
# include_service_catalog = False
#
# [filter:keystoneauth]
# use = egg:swift#keystoneauth
# Operator roles is the role which user would be allowed to manage a
# tenant and be able to create container or give ACL to others.
# operator_roles = admin, swiftoperator
# The reseller admin role has the ability to create and delete accounts
# reseller_admin_role = ResellerAdmin

[filter:healthcheck]
use = egg:swift#healthcheck
# An optional filesystem path, which if present, will cause the healthcheck
# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE".
# This facility may be used to temporarily remove a Swift node from a load
# balancer pool during maintenance or upgrade (remove the file to allow the
# node back into the load balancer pool).
# disable_path =

[filter:cache]
use = egg:swift#memcache
# You can override the default log routing for this filter here:
# set log_name = cache
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log
#
# If not set here, the value for memcache_servers will be read from
# memcache.conf (see memcache.conf-sample) or lacking that file, it will
# default to the value below. You can specify multiple servers separated with
# commas, as in: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211
#
# Sets how memcache values are serialized and deserialized:
# 0 = older, insecure pickle serialization
# 1 = json serialization but pickles can still be read (still insecure)
# 2 = json serialization only (secure and the default)
# If not set here, the value for memcache_serialization_support will be read
# from /etc/swift/memcache.conf (see memcache.conf-sample).
# To avoid an instant full cache flush, existing installations should
# upgrade with 0, then set to 1 and reload, then after some time (24 hours)
# set to 2 and reload.
# In the future, the ability to use pickle serialization will be removed.
# memcache_serialization_support = 2
#
# Sets the maximum number of connections to each memcached server per worker
# memcache_max_connections = 2

[filter:ratelimit]
use = egg:swift#ratelimit
# You can override the default log routing for this filter here:
# set log_name = ratelimit
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log
#
# clock_accuracy should represent how accurate the proxy servers' system clocks
# are with each other. 1000 means that all the proxies' clock are accurate to
# each other within 1 millisecond.  No ratelimit should be higher than the
# clock accuracy.
# clock_accuracy = 1000
#
# max_sleep_time_seconds = 60
#
# log_sleep_time_seconds of 0 means disabled
# log_sleep_time_seconds = 0
#
# allows for slow rates (e.g. running up to 5 sec's behind) to catch up.
# rate_buffer_seconds = 5
#
# account_ratelimit of 0 means disabled
# account_ratelimit = 0

# these are comma separated lists of account names
# account_whitelist = a,b
# account_blacklist = c,d

# with container_limit_x = r
# for containers of size x limit write requests per second to r.  The container
# rate will be linearly interpolated from the values given. With the values
# below, a container of size 5 will get a rate of 75.
# container_ratelimit_0 = 100
# container_ratelimit_10 = 50
# container_ratelimit_50 = 20

# Similarly to the above container-level write limits, the following will limit
# container GET (listing) requests.
# container_listing_ratelimit_0 = 100
# container_listing_ratelimit_10 = 50
# container_listing_ratelimit_50 = 20

[filter:domain_remap]
use = egg:swift#domain_remap
# You can override the default log routing for this filter here:
# set log_name = domain_remap
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log
#
# storage_domain = example.com
# path_root = v1
# reseller_prefixes = AUTH

[filter:catch_errors]
use = egg:swift#catch_errors
# You can override the default log routing for this filter here:
# set log_name = catch_errors
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log

[filter:cname_lookup]
# Note: this middleware requires python-dnspython
use = egg:swift#cname_lookup
# You can override the default log routing for this filter here:
# set log_name = cname_lookup
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log
#
# Specify the storage_domain that match your cloud, multiple domains
# can be specified separated by a comma
# storage_domain = example.com
#
# lookup_depth = 1

# Note: Put staticweb just after your auth filter(s) in the pipeline
[filter:staticweb]
use = egg:swift#staticweb

# Note: Put tempurl before dlo, slo and your auth filter(s) in the pipeline
[filter:tempurl]
use = egg:swift#tempurl
# The methods allowed with Temp URLs.
# methods = GET HEAD PUT
#
# The headers to remove from incoming requests. Simply a whitespace delimited
# list of header names and names can optionally end with '*' to indicate a
# prefix match. incoming_allow_headers is a list of exceptions to these
# removals.
# incoming_remove_headers = x-timestamp
#
# The headers allowed as exceptions to incoming_remove_headers. Simply a
# whitespace delimited list of header names and names can optionally end with
# '*' to indicate a prefix match.
# incoming_allow_headers =
#
# The headers to remove from outgoing responses. Simply a whitespace delimited
# list of header names and names can optionally end with '*' to indicate a
# prefix match. outgoing_allow_headers is a list of exceptions to these
# removals.
# outgoing_remove_headers = x-object-meta-*
#
# The headers allowed as exceptions to outgoing_remove_headers. Simply a
# whitespace delimited list of header names and names can optionally end with
# '*' to indicate a prefix match.
# outgoing_allow_headers = x-object-meta-public-*

# Note: Put formpost just before your auth filter(s) in the pipeline
[filter:formpost]
use = egg:swift#formpost

# Note: Just needs to be placed before the proxy-server in the pipeline.
[filter:name_check]
use = egg:swift#name_check
# forbidden_chars = '"`<>
# maximum_length = 255
# forbidden_regexp = /\./|/\.\./|/\.$|/\.\.$

[filter:list-endpoints]
use = egg:swift#list_endpoints
# list_endpoints_path = /endpoints/

[filter:proxy-logging]
use = egg:swift#proxy_logging
# If not set, logging directives from [DEFAULT] without "access_" will be used
# access_log_name = swift
# access_log_facility = LOG_LOCAL0
# access_log_level = INFO
# access_log_address = /dev/log
#
# If set, access_log_udp_host will override access_log_address
# access_log_udp_host =
# access_log_udp_port = 514
#
# You can use log_statsd_* from [DEFAULT] or override them here:
# access_log_statsd_host = localhost
# access_log_statsd_port = 8125
# access_log_statsd_default_sample_rate = 1.0
# access_log_statsd_sample_rate_factor = 1.0
# access_log_statsd_metric_prefix =
# access_log_headers = false
#
# If access_log_headers is True and access_log_headers_only is set only
# these headers are logged. Multiple headers can be defined as comma separated
# list like this: access_log_headers_only = Host, X-Object-Meta-Mtime
# access_log_headers_only =
#
# By default, the X-Auth-Token is logged. To obscure the value,
# set reveal_sensitive_prefix to the number of characters to log.
# For example, if set to 12, only the first 12 characters of the
# token appear in the log. An unauthorized access of the log file
# won't allow unauthorized usage of the token. However, the first
# 12 or so characters is unique enough that you can trace/debug
# token usage. Set to 0 to suppress the token completely (replaced
# by '...' in the log).
# Note: reveal_sensitive_prefix will not affect the value
# logged with access_log_headers=True.
# reveal_sensitive_prefix = 8192
#
# What HTTP methods are allowed for StatsD logging (comma-sep); request methods
# not in this list will have "BAD_METHOD" for the  portion of the metric.
# log_statsd_valid_http_methods = GET,HEAD,POST,PUT,DELETE,COPY,OPTIONS
#
# Note: The double proxy-logging in the pipeline is not a mistake. The
# left-most proxy-logging is there to log requests that were handled in
# middleware and never made it through to the right-most middleware (and
# proxy server). Double logging is prevented for normal requests. See
# proxy-logging docs.

# Note: Put before both ratelimit and auth in the pipeline.
[filter:bulk]
use = egg:swift#bulk
# max_containers_per_extraction = 10000
# max_failed_extractions = 1000
# max_deletes_per_request = 10000
# max_failed_deletes = 1000

# In order to keep a connection active during a potentially long bulk request,
# Swift may return whitespace prepended to the actual response body. This
# whitespace will be yielded no more than every yield_frequency seconds.
# yield_frequency = 10

# Note: The following parameter is used during a bulk delete of objects and
# their container. This would frequently fail because it is very likely
# that all replicated objects have not been deleted by the time the middleware got a
# successful response. It can be configured the number of retries. And the
# number of seconds to wait between each retry will be 1.5**retry

# delete_container_retry_count = 0

# Note: Put after auth in the pipeline.
[filter:container-quotas]
use = egg:swift#container_quotas

# Note: Put before both ratelimit and auth in the pipeline.
[filter:slo]
use = egg:swift#slo
# max_manifest_segments = 1000
# max_manifest_size = 2097152
# min_segment_size = 1048576
# Start rate-limiting SLO segment serving after the Nth segment of a
# segmented object.
# rate_limit_after_segment = 10
#
# Once segment rate-limiting kicks in for an object, limit segments served
# to N per second. 0 means no rate-limiting.
# rate_limit_segments_per_sec = 0
#
# Time limit on GET requests (seconds)
# max_get_time = 86400

# Note: Put before both ratelimit and auth in the pipeline, but after
# gatekeeper, catch_errors, and proxy_logging (the first instance).
# If you don't put it in the pipeline, it will be inserted for you.
[filter:dlo]
use = egg:swift#dlo
# Start rate-limiting DLO segment serving after the Nth segment of a
# segmented object.
# rate_limit_after_segment = 10
#
# Once segment rate-limiting kicks in for an object, limit segments served
# to N per second. 0 means no rate-limiting.
# rate_limit_segments_per_sec = 1
#
# Time limit on GET requests (seconds)
# max_get_time = 86400

[filter:account-quotas]
use = egg:swift#account_quotas

[filter:gatekeeper]
use = egg:swift#gatekeeper
# You can override the default log routing for this filter here:
# set log_name = gatekeeper
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_headers = false
# set log_address = /dev/log

[filter:container_sync]
use = egg:swift#container_sync
# Set this to false if you want to disallow any full url values to be set for
# any new X-Container-Sync-To headers. This will keep any new full urls from
# coming in, but won't change any existing values already in the cluster.
# Updating those will have to be done manually, as knowing what the true realm
# endpoint should be cannot always be guessed.
# allow_full_urls = true
swift-1.13.1/etc/dispersion.conf-sample0000664000175400017540000000110712323703611021144 0ustar  jenkinsjenkins00000000000000[dispersion]
# Please create a new account solely for using dispersion tools, which is
# helpful for keep your own data clean.
auth_url = http://localhost:8080/auth/v1.0
auth_user = test:tester
auth_key = testing
# auth_url = http://localhost:5000/v2.0/
# auth_user = tenant:user
# auth_key = password
# auth_version = 2.0
# endpoint_type = publicURL
# keystone_api_insecure = no
#
# swift_dir = /etc/swift
# dispersion_coverage = 1.0
# retries = 5
# concurrency = 25
# container_populate = yes
# object_populate = yes
# container_report = yes
# object_report = yes
# dump_json = no
swift-1.13.1/etc/memcache.conf-sample0000664000175400017540000000161012323703611020526 0ustar  jenkinsjenkins00000000000000[memcache]
# You can use this single conf file instead of having memcache_servers set in
# several other conf files under [filter:cache] for example. You can specify
# multiple servers separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211
# memcache_servers = 127.0.0.1:11211
#
# Sets how memcache values are serialized and deserialized:
# 0 = older, insecure pickle serialization
# 1 = json serialization but pickles can still be read (still insecure)
# 2 = json serialization only (secure and the default)
# To avoid an instant full cache flush, existing installations should
# upgrade with 0, then set to 1 and reload, then after some time (24 hours)
# set to 2 and reload.
# In the future, the ability to use pickle serialization will be removed.
# memcache_serialization_support = 2
#
# Sets the maximum number of connections to each memcached server per worker
# memcache_max_connections = 2
swift-1.13.1/etc/container-sync-realms.conf-sample0000664000175400017540000000362012323703611023204 0ustar  jenkinsjenkins00000000000000# [DEFAULT]
# The number of seconds between checking the modified time of this config file
# for changes and therefore reloading it.
# mtime_check_interval = 300


# [realm1]
# key = realm1key
# key2 = realm1key2
# cluster_name1 = https://host1/v1/
# cluster_name2 = https://host2/v1/
#
# [realm2]
# key = realm2key
# key2 = realm2key2
# cluster_name3 = https://host3/v1/
# cluster_name4 = https://host4/v1/


# Each section name is the name of a sync realm. A sync realm is a set of
# clusters that have agreed to allow container syncing with each other. Realm
# names will be considered case insensitive.
#
# The key is the overall cluster-to-cluster key used in combination with the
# external users' key that they set on their containers' X-Container-Sync-Key
# metadata header values. These keys will be used to sign each request the
# container sync daemon makes and used to validate each incoming container sync
# request.
#
# The key2 is optional and is an additional key incoming requests will be
# checked against. This is so you can rotate keys if you wish; you move the
# existing key to key2 and make a new key value.
#
# Any values in the realm section whose names begin with cluster_ will indicate
# the name and endpoint of a cluster and will be used by external users in
# their containers' X-Container-Sync-To metadata header values with the format
# "realm_name/cluster_name/container_name". Realm and cluster names are
# considered case insensitive.
#
# The endpoint is what the container sync daemon will use when sending out
# requests to that cluster. Keep in mind this endpoint must be reachable by all
# container servers, since that is where the container sync daemon runs. Note
# the the endpoint ends with /v1/ and that the container sync daemon will then
# add the account/container/obj name after that.
#
# Distribute this container-sync-realms.conf file to all your proxy servers
# and container servers.
swift-1.13.1/etc/rsyncd.conf-sample0000664000175400017540000000060712323703611020273 0ustar  jenkinsjenkins00000000000000uid = swift
gid = swift
log file = /var/log/rsyncd.log
pid file = /var/run/rsyncd.pid

[account]
max connections = 2
path = /srv/node
read only = false
lock file = /var/lock/account.lock

[container]
max connections = 4
path = /srv/node
read only = false
lock file = /var/lock/container.lock

[object]
max connections = 8
path = /srv/node
read only = false
lock file = /var/lock/object.lock
swift-1.13.1/etc/account-server.conf-sample0000664000175400017540000001072312323703611021731 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6002
# bind_timeout = 30
# backlog = 4096
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
# disable_fallocate = false
#
# Use an integer to override the number of pre-forked processes that will
# accept connections.
# workers = auto
#
# Maximum concurrent requests per worker
# max_clients = 1024
#
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
#
# If you don't mind the extra disk space usage in overhead, you can turn this
# on to preallocate disk space with SQLite databases to decrease fragmentation.
# db_preallocation = off
#
# eventlet_debug = false
#
# You can set fallocate_reserve to the number of bytes you'd like fallocate to
# reserve, whether there is space for the given file size or not.
# fallocate_reserve = 0

[pipeline:main]
pipeline = healthcheck recon account-server

[app:account-server]
use = egg:swift#account
# You can override the default log routing for this app here:
# set log_name = account-server
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = true
# set log_address = /dev/log
#
# auto_create_account_prefix = .
#
# Configure parameter for creating specific server
# To handle all verbs, including replication verbs, do not specify
# "replication_server" (this is the default). To only handle replication,
# set to a True value (e.g. "True" or "1"). To handle only non-replication
# verbs, set to "False". Unless you have a separate replication network, you
# should not specify any value for "replication_server".
# replication_server = false

[filter:healthcheck]
use = egg:swift#healthcheck
# An optional filesystem path, which if present, will cause the healthcheck
# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE"
# disable_path =

[filter:recon]
use = egg:swift#recon
# recon_cache_path = /var/cache/swift

[account-replicator]
# You can override the default log routing for this app here (don't use set!):
# log_name = account-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
# concurrency = 8
# interval = 30
#
# How long without an error before a node's error count is reset. This will
# also be how long before a node is reenabled after suppression is triggered.
# error_suppression_interval = 60
#
# How many errors can accumulate before a node is temporarily ignored.
# error_suppression_limit = 10
#
# node_timeout = 10
# conn_timeout = 0.5
#
# The replicator also performs reclamation
# reclaim_age = 604800
#
# Time in seconds to wait between replication passes
# run_pause = 30
#
# recon_cache_path = /var/cache/swift

[account-auditor]
# You can override the default log routing for this app here (don't use set!):
# log_name = account-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# Will audit each account at most once per interval
# interval = 1800
#
# log_facility = LOG_LOCAL0
# log_level = INFO
# accounts_per_second = 200
# recon_cache_path = /var/cache/swift

[account-reaper]
# You can override the default log routing for this app here (don't use set!):
# log_name = account-reaper
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# concurrency = 25
# interval = 3600
# node_timeout = 10
# conn_timeout = 0.5
#
# Normally, the reaper begins deleting account information for deleted accounts
# immediately; you can set this to delay its work however. The value is in
# seconds; 2592000 = 30 days for example.
# delay_reaping = 0
#
# If the account fails to be be reaped due to a persistent error, the
# account reaper will log a message such as:
#     Account  has not been reaped since 
# You can search logs for this message if space is not being reclaimed
# after you delete account(s).
# Default is 2592000 seconds (30 days). This is in addition to any time
# requested by delay_reaping.
# reap_warn_after = 2592000
swift-1.13.1/etc/object-server.conf-sample0000664000175400017540000001760612323703611021552 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6000
# bind_timeout = 30
# backlog = 4096
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
# disable_fallocate = false
# expiring_objects_container_divisor = 86400
# expiring_objects_account_name = expiring_objects
#
# Use an integer to override the number of pre-forked processes that will
# accept connections.
# workers = auto
#
# Maximum concurrent requests per worker
# max_clients = 1024
#
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
#
# eventlet_debug = false
#
# You can set fallocate_reserve to the number of bytes you'd like fallocate to
# reserve, whether there is space for the given file size or not.
# fallocate_reserve = 0
#
# Time to wait while attempting to connect to another backend node.
# conn_timeout = 0.5
# Time to wait while sending each chunk of data to another backend node.
# node_timeout = 3
# Time to wait while receiving each chunk of data from a client or another
# backend node.
# client_timeout = 60
#
# network_chunk_size = 65536
# disk_chunk_size = 65536

[pipeline:main]
pipeline = healthcheck recon object-server

[app:object-server]
use = egg:swift#object
# You can override the default log routing for this app here:
# set log_name = object-server
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = true
# set log_address = /dev/log
#
# max_upload_time = 86400
# slow = 0
#
# Objects smaller than this are not evicted from the buffercache once read
# keep_cache_size = 5424880
#
# If true, objects for authenticated GET requests may be kept in buffer cache
# if small enough
# keep_cache_private = false
#
# on PUTs, sync data every n MB
# mb_per_sync = 512
#
# Comma separated list of headers that can be set in metadata on an object.
# This list is in addition to X-Object-Meta-* headers and cannot include
# Content-Type, etag, Content-Length, or deleted
# allowed_headers = Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object
#
# auto_create_account_prefix = .
#
# A value of 0 means "don't use thread pools". A reasonable starting point is
# 4.
# threads_per_disk = 0
#
# Configure parameter for creating specific server
# To handle all verbs, including replication verbs, do not specify
# "replication_server" (this is the default). To only handle replication,
# set to a True value (e.g. "True" or "1"). To handle only non-replication
# verbs, set to "False". Unless you have a separate replication network, you
# should not specify any value for "replication_server".
# replication_server = false
#
# Set to restrict the number of concurrent incoming REPLICATION requests
# Set to 0 for unlimited
# Note that REPLICATION is currently an ssync only item
# replication_concurrency = 4
#
# Restricts incoming REPLICATION requests to one per device,
# replication_currency above allowing. This can help control I/O to each
# device, but you may wish to set this to False to allow multiple REPLICATION
# requests (up to the above replication_concurrency setting) per device.
# replication_one_per_device = True
#
# Number of seconds to wait for an existing replication device lock before
# giving up.
# replication_lock_timeout = 15
#
# These next two settings control when the REPLICATION subrequest handler will
# abort an incoming REPLICATION attempt. An abort will occur if there are at
# least threshold number of failures and the value of failures / successes
# exceeds the ratio. The defaults of 100 and 1.0 means that at least 100
# failures have to occur and there have to be more failures than successes for
# an abort to occur.
# replication_failure_threshold = 100
# replication_failure_ratio = 1.0

[filter:healthcheck]
use = egg:swift#healthcheck
# An optional filesystem path, which if present, will cause the healthcheck
# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE"
# disable_path =

[filter:recon]
use = egg:swift#recon
#recon_cache_path = /var/cache/swift
#recon_lock_path = /var/lock

[object-replicator]
# You can override the default log routing for this app here (don't use set!):
# log_name = object-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# daemonize = on
# run_pause = 30
# concurrency = 1
# stats_interval = 300
#
# The sync method to use; default is rsync but you can use ssync to try the
# EXPERIMENTAL all-swift-code-no-rsync-callouts method. Once ssync is verified
# as having performance comparable to, or better than, rsync, we plan to
# deprecate rsync so we can move on with more features for replication.
# sync_method = rsync
#
# max duration of a partition rsync
# rsync_timeout = 900
#
# bandwidth limit for rsync in kB/s. 0 means unlimited
# rsync_bwlimit = 0
#
# passed to rsync for io op timeout
# rsync_io_timeout = 30
#
# node_timeout = 
# max duration of an http request; this is for REPLICATE finalization calls and
# so should be longer than node_timeout
# http_timeout = 60
#
# attempts to kill all workers if nothing replicates for lockup_timeout seconds
# lockup_timeout = 1800
#
# The replicator also performs reclamation
# reclaim_age = 604800
#
# ring_check_interval = 15
# recon_cache_path = /var/cache/swift
#
# limits how long rsync error log lines are
# 0 means to log the entire line
# rsync_error_log_line_length = 0
#
# handoffs_first and handoff_delete are options for a special case
# such as disk full in the cluster. These two options SHOULD NOT BE
# CHANGED, except for such an extreme situations. (e.g. disks filled up
# or are about to fill up. Anyway, DO NOT let your drives fill up)
# handoffs_first is the flag to replicate handoffs prior to canonical
# partitions. It allows to force syncing and deleting handoffs quickly.
# If set to a True value(e.g. "True" or "1"), partitions
# that are not supposed to be on the node will be replicated first.
# handoffs_first = False
#
# handoff_delete is the number of replicas which are ensured in swift.
# If the number less than the number of replicas is set, object-replicator
# could delete local handoffs even if all replicas are not ensured in the
# cluster. Object-replicator would remove local handoff partition directories
# after syncing partition when the number of successful responses is greater
# than or equal to this number. By default(auto), handoff partitions will be
# removed  when it has successfully replicated to all the canonical nodes.
# handoff_delete = auto

[object-updater]
# You can override the default log routing for this app here (don't use set!):
# log_name = object-updater
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# interval = 300
# concurrency = 1
# node_timeout = 
# slowdown will sleep that amount between objects
# slowdown = 0.01
#
# recon_cache_path = /var/cache/swift

[object-auditor]
# You can override the default log routing for this app here (don't use set!):
# log_name = object-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# files_per_second = 20
# bytes_per_second = 10000000
# log_time = 3600
# zero_byte_files_per_second = 50
# recon_cache_path = /var/cache/swift

# Takes a comma separated list of ints. If set, the object auditor will
# increment a counter for every object whose size is <= to the given break
# points and report the result after a full scan.
# object_size_stats =
swift-1.13.1/etc/container-server.conf-sample0000664000175400017540000001134012323703611022253 0ustar  jenkinsjenkins00000000000000[DEFAULT]
# bind_ip = 0.0.0.0
# bind_port = 6001
# bind_timeout = 30
# backlog = 4096
# user = swift
# swift_dir = /etc/swift
# devices = /srv/node
# mount_check = true
# disable_fallocate = false
#
# Use an integer to override the number of pre-forked processes that will
# accept connections.
# workers = auto
#
# Maximum concurrent requests per worker
# max_clients = 1024
#
# This is a comma separated list of hosts allowed in the X-Container-Sync-To
# field for containers. This is the old-style of using container sync. It is
# strongly recommended to use the new style of a separate
# container-sync-realms.conf -- see container-sync-realms.conf-sample
# allowed_sync_hosts = 127.0.0.1
#
# You can specify default log routing here if you want:
# log_name = swift
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# comma separated list of functions to call to setup custom log handlers.
# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
# adapted_logger
# log_custom_handlers =
#
# If set, log_udp_host will override log_address
# log_udp_host =
# log_udp_port = 514
#
# You can enable StatsD logging here:
# log_statsd_host = localhost
# log_statsd_port = 8125
# log_statsd_default_sample_rate = 1.0
# log_statsd_sample_rate_factor = 1.0
# log_statsd_metric_prefix =
#
# If you don't mind the extra disk space usage in overhead, you can turn this
# on to preallocate disk space with SQLite databases to decrease fragmentation.
# db_preallocation = off
#
# eventlet_debug = false
#
# You can set fallocate_reserve to the number of bytes you'd like fallocate to
# reserve, whether there is space for the given file size or not.
# fallocate_reserve = 0

[pipeline:main]
pipeline = healthcheck recon container-server

[app:container-server]
use = egg:swift#container
# You can override the default log routing for this app here:
# set log_name = container-server
# set log_facility = LOG_LOCAL0
# set log_level = INFO
# set log_requests = true
# set log_address = /dev/log
#
# node_timeout = 3
# conn_timeout = 0.5
# allow_versions = false
# auto_create_account_prefix = .
#
# Configure parameter for creating specific server
# To handle all verbs, including replication verbs, do not specify
# "replication_server" (this is the default). To only handle replication,
# set to a True value (e.g. "True" or "1"). To handle only non-replication
# verbs, set to "False". Unless you have a separate replication network, you
# should not specify any value for "replication_server".
# replication_server = false

[filter:healthcheck]
use = egg:swift#healthcheck
# An optional filesystem path, which if present, will cause the healthcheck
# URL to return "503 Service Unavailable" with a body of "DISABLED BY FILE"
# disable_path =

[filter:recon]
use = egg:swift#recon
#recon_cache_path = /var/cache/swift

[container-replicator]
# You can override the default log routing for this app here (don't use set!):
# log_name = container-replicator
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# vm_test_mode = no
# per_diff = 1000
# max_diffs = 100
# concurrency = 8
# interval = 30
# node_timeout = 10
# conn_timeout = 0.5
#
# The replicator also performs reclamation
# reclaim_age = 604800
#
# Time in seconds to wait between replication passes
# run_pause = 30
#
# recon_cache_path = /var/cache/swift

[container-updater]
# You can override the default log routing for this app here (don't use set!):
# log_name = container-updater
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# interval = 300
# concurrency = 4
# node_timeout = 3
# conn_timeout = 0.5
#
# slowdown will sleep that amount between containers
# slowdown = 0.01
#
# Seconds to suppress updating an account that has generated an error
# account_suppression_time = 60
#
# recon_cache_path = /var/cache/swift

[container-auditor]
# You can override the default log routing for this app here (don't use set!):
# log_name = container-auditor
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# Will audit each container at most once per interval
# interval = 1800
#
# containers_per_second = 200
# recon_cache_path = /var/cache/swift

[container-sync]
# You can override the default log routing for this app here (don't use set!):
# log_name = container-sync
# log_facility = LOG_LOCAL0
# log_level = INFO
# log_address = /dev/log
#
# If you need to use an HTTP Proxy, set it here; defaults to no proxy.
# You can also set this to a comma separated list of HTTP Proxies and they will
# be randomly used (simple load balancing).
# sync_proxy = http://10.1.1.1:8888,http://10.1.1.2:8888
#
# Will sync each container at most once per interval
# interval = 300
#
# Maximum amount of time to spend syncing each container per pass
# container_time = 60
swift-1.13.1/swift.egg-info/0000775000175400017540000000000012323703665016724 5ustar  jenkinsjenkins00000000000000swift-1.13.1/swift.egg-info/dependency_links.txt0000664000175400017540000000000112323703665022772 0ustar  jenkinsjenkins00000000000000
swift-1.13.1/swift.egg-info/not-zip-safe0000664000175400017540000000000112323703663021150 0ustar  jenkinsjenkins00000000000000
swift-1.13.1/swift.egg-info/PKG-INFO0000664000175400017540000001062512323703665020025 0ustar  jenkinsjenkins00000000000000Metadata-Version: 1.1
Name: swift
Version: 1.13.1
Summary: OpenStack Object Storage
Home-page: http://www.openstack.org/
Author: OpenStack
Author-email: openstack-dev@lists.openstack.org
License: UNKNOWN
Description: # Swift
        
        A distributed object storage system designed to scale from a single machine
        to thousands of servers. Swift is optimized for multi-tenancy and high
        concurrency. Swift is ideal for backups, web and mobile content, and any other
        unstructured data that can grow without bound.
        
        Swift provides a simple, REST-based API fully documented at
        http://docs.openstack.org/.
        
        Swift was originally developed as the basis for Rackspace's Cloud Files and
        was open-sourced in 2010 as part of the OpenStack project. It has since grown
        to include contributions from many companies and has spawned a thriving
        ecosystem of 3rd party tools. Swift's contributors are listed in the AUTHORS
        file.
        
        ## Docs
        
        To build documentation install sphinx (`pip install sphinx`), run
        `python setup.py build_sphinx`, and then browse to /doc/build/html/index.html.
        These docs are auto-generated after every commit and available online at
        http://docs.openstack.org/developer/swift/.
        
        ## For Developers
        
        The best place to get started is the ["SAIO - Swift All In One"](http://docs.openstack.org/developer/swift/development_saio.html).
        This document will walk you through setting up a development cluster of Swift
        in a VM. The SAIO environment is ideal for running small-scale tests against
        swift and trying out new features and bug fixes.
        
        You can run unit tests with `.unittests` and functional tests with
        `.functests`.
        
        ### Code Organization
        
         * bin/: Executable scripts that are the processes run by the deployer
         * doc/: Documentation
         * etc/: Sample config files
         * swift/: Core code
            * account/: account server
            * common/: code shared by different modules
                * middleware/: "standard", officially-supported middleware
                * ring/: code implementing Swift's ring
            * container/: container server
            * obj/: object server
            * proxy/: proxy server
         * test/: Unit and functional tests
        
        ### Data Flow
        
        Swift is a WSGI application and uses eventlet's WSGI server. After the
        processes are running, the entry point for new requests is the `Application`
        class in `swift/proxy/server.py`. From there, a controller is chosen, and the
        request is processed. The proxy may choose to forward the request to a back-
        end server. For example, the entry point for requests to the object server is
        the `ObjectController` class in `swift/obj/server.py`.
        
        
        ## For Deployers
        
        Deployer docs are also available at
        http://docs.openstack.org/developer/swift/. A good starting point is at
        http://docs.openstack.org/developer/swift/deployment_guide.html
        
        You can run functional tests against a swift cluster with `.functests`. These
        functional tests require `/etc/swift/test.conf` to run. A sample config file
        can be found in this source tree in `test/sample.conf`.
        
        ## For Client Apps
        
        For client applications, official Python language bindings are provided at
        http://github.com/openstack/python-swiftclient.
        
        Complete API documentation at
        http://docs.openstack.org/api/openstack-object-storage/1.0/content/
        
        ----
        
        For more information come hang out in #openstack-swift on freenode.
        
        Thanks,
        
        The Swift Development Team
        
        
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Environment :: OpenStack
Classifier: Intended Audience :: Information Technology
Classifier: Intended Audience :: System Administrators
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Operating System :: POSIX :: Linux
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.6
Classifier: Programming Language :: Python :: 2.7
swift-1.13.1/swift.egg-info/entry_points.txt0000664000175400017540000000324112323703665022222 0ustar  jenkinsjenkins00000000000000[paste.app_factory]
account = swift.account.server:app_factory
container = swift.container.server:app_factory
mem_object = swift.obj.mem_server:app_factory
object = swift.obj.server:app_factory
proxy = swift.proxy.server:app_factory

[paste.filter_factory]
account_quotas = swift.common.middleware.account_quotas:filter_factory
bulk = swift.common.middleware.bulk:filter_factory
catch_errors = swift.common.middleware.catch_errors:filter_factory
cname_lookup = swift.common.middleware.cname_lookup:filter_factory
container_quotas = swift.common.middleware.container_quotas:filter_factory
container_sync = swift.common.middleware.container_sync:filter_factory
crossdomain = swift.common.middleware.crossdomain:filter_factory
dlo = swift.common.middleware.dlo:filter_factory
domain_remap = swift.common.middleware.domain_remap:filter_factory
formpost = swift.common.middleware.formpost:filter_factory
gatekeeper = swift.common.middleware.gatekeeper:filter_factory
healthcheck = swift.common.middleware.healthcheck:filter_factory
keystoneauth = swift.common.middleware.keystoneauth:filter_factory
list_endpoints = swift.common.middleware.list_endpoints:filter_factory
memcache = swift.common.middleware.memcache:filter_factory
name_check = swift.common.middleware.name_check:filter_factory
proxy_logging = swift.common.middleware.proxy_logging:filter_factory
ratelimit = swift.common.middleware.ratelimit:filter_factory
recon = swift.common.middleware.recon:filter_factory
slo = swift.common.middleware.slo:filter_factory
staticweb = swift.common.middleware.staticweb:filter_factory
tempauth = swift.common.middleware.tempauth:filter_factory
tempurl = swift.common.middleware.tempurl:filter_factory

swift-1.13.1/swift.egg-info/top_level.txt0000664000175400017540000000000612323703665021452 0ustar  jenkinsjenkins00000000000000swift
swift-1.13.1/swift.egg-info/requires.txt0000664000175400017540000000016012323703665021321 0ustar  jenkinsjenkins00000000000000dnspython>=1.9.4
eventlet>=0.9.15
greenlet>=0.3.1
netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4swift-1.13.1/swift.egg-info/SOURCES.txt0000664000175400017540000002616612323703665020623 0ustar  jenkinsjenkins00000000000000.coveragerc
.functests
.mailmap
.probetests
.unittests
AUTHORS
CHANGELOG
CONTRIBUTING.md
LICENSE
MANIFEST.in
README.md
babel.cfg
requirements.txt
setup.cfg
setup.py
test-requirements.txt
tox.ini
bin/swift-account-audit
bin/swift-account-auditor
bin/swift-account-info
bin/swift-account-reaper
bin/swift-account-replicator
bin/swift-account-server
bin/swift-config
bin/swift-container-auditor
bin/swift-container-info
bin/swift-container-replicator
bin/swift-container-server
bin/swift-container-sync
bin/swift-container-updater
bin/swift-dispersion-populate
bin/swift-dispersion-report
bin/swift-drive-audit
bin/swift-form-signature
bin/swift-get-nodes
bin/swift-init
bin/swift-object-auditor
bin/swift-object-expirer
bin/swift-object-info
bin/swift-object-replicator
bin/swift-object-server
bin/swift-object-updater
bin/swift-oldies
bin/swift-orphans
bin/swift-proxy-server
bin/swift-recon
bin/swift-recon-cron
bin/swift-ring-builder
bin/swift-temp-url
doc/manpages/account-server.conf.5
doc/manpages/container-server.conf.5
doc/manpages/dispersion.conf.5
doc/manpages/object-expirer.conf.5
doc/manpages/object-server.conf.5
doc/manpages/proxy-server.conf.5
doc/manpages/swift-account-auditor.1
doc/manpages/swift-account-info.1
doc/manpages/swift-account-reaper.1
doc/manpages/swift-account-replicator.1
doc/manpages/swift-account-server.1
doc/manpages/swift-container-auditor.1
doc/manpages/swift-container-info.1
doc/manpages/swift-container-replicator.1
doc/manpages/swift-container-server.1
doc/manpages/swift-container-sync.1
doc/manpages/swift-container-updater.1
doc/manpages/swift-dispersion-populate.1
doc/manpages/swift-dispersion-report.1
doc/manpages/swift-get-nodes.1
doc/manpages/swift-init.1
doc/manpages/swift-object-auditor.1
doc/manpages/swift-object-expirer.1
doc/manpages/swift-object-info.1
doc/manpages/swift-object-replicator.1
doc/manpages/swift-object-server.1
doc/manpages/swift-object-updater.1
doc/manpages/swift-orphans.1
doc/manpages/swift-proxy-server.1
doc/manpages/swift-recon.1
doc/manpages/swift-ring-builder.1
doc/saio/rsyncd.conf
doc/saio/bin/remakerings
doc/saio/bin/resetswift
doc/saio/bin/startmain
doc/saio/bin/startrest
doc/saio/rsyslog.d/10-swift.conf
doc/saio/swift/object-expirer.conf
doc/saio/swift/proxy-server.conf
doc/saio/swift/swift.conf
doc/saio/swift/account-server/1.conf
doc/saio/swift/account-server/2.conf
doc/saio/swift/account-server/3.conf
doc/saio/swift/account-server/4.conf
doc/saio/swift/container-server/1.conf
doc/saio/swift/container-server/2.conf
doc/saio/swift/container-server/3.conf
doc/saio/swift/container-server/4.conf
doc/saio/swift/object-server/1.conf
doc/saio/swift/object-server/2.conf
doc/saio/swift/object-server/3.conf
doc/saio/swift/object-server/4.conf
doc/source/account.rst
doc/source/admin_guide.rst
doc/source/apache_deployment_guide.rst
doc/source/associated_projects.rst
doc/source/conf.py
doc/source/container.rst
doc/source/cors.rst
doc/source/crossdomain.rst
doc/source/db.rst
doc/source/deployment_guide.rst
doc/source/development_auth.rst
doc/source/development_guidelines.rst
doc/source/development_middleware.rst
doc/source/development_ondisk_backends.rst
doc/source/development_saio.rst
doc/source/getting_started.rst
doc/source/howto_installmultinode.rst
doc/source/index.rst
doc/source/logs.rst
doc/source/middleware.rst
doc/source/misc.rst
doc/source/object.rst
doc/source/overview_architecture.rst
doc/source/overview_auth.rst
doc/source/overview_container_sync.rst
doc/source/overview_expiring_objects.rst
doc/source/overview_large_objects.rst
doc/source/overview_object_versioning.rst
doc/source/overview_reaper.rst
doc/source/overview_replication.rst
doc/source/overview_ring.rst
doc/source/proxy.rst
doc/source/ratelimit.rst
doc/source/replication_network.rst
doc/source/ring.rst
doc/source/test-cors.html
doc/source/_ga/layout.html
doc/source/_static/basic.css
doc/source/_static/default.css
doc/source/_static/tweaks.css
doc/source/_theme/layout.html
doc/source/_theme/theme.conf
etc/account-server.conf-sample
etc/container-server.conf-sample
etc/container-sync-realms.conf-sample
etc/dispersion.conf-sample
etc/drive-audit.conf-sample
etc/memcache.conf-sample
etc/mime.types-sample
etc/object-expirer.conf-sample
etc/object-server.conf-sample
etc/proxy-server.conf-sample
etc/rsyncd.conf-sample
etc/swift-rsyslog.conf-sample
etc/swift.conf-sample
examples/apache2/account-server.template
examples/apache2/container-server.template
examples/apache2/object-server.template
examples/apache2/proxy-server.template
examples/wsgi/account-server.wsgi.template
examples/wsgi/container-server.wsgi.template
examples/wsgi/object-server.wsgi.template
examples/wsgi/proxy-server.wsgi.template
locale/swift.pot
swift/__init__.py
swift.egg-info/PKG-INFO
swift.egg-info/SOURCES.txt
swift.egg-info/dependency_links.txt
swift.egg-info/entry_points.txt
swift.egg-info/not-zip-safe
swift.egg-info/requires.txt
swift.egg-info/top_level.txt
swift/account/__init__.py
swift/account/auditor.py
swift/account/backend.py
swift/account/reaper.py
swift/account/replicator.py
swift/account/server.py
swift/account/utils.py
swift/cli/__init__.py
swift/cli/info.py
swift/cli/recon.py
swift/cli/ringbuilder.py
swift/common/__init__.py
swift/common/bufferedhttp.py
swift/common/constraints.py
swift/common/container_sync_realms.py
swift/common/daemon.py
swift/common/db.py
swift/common/db_replicator.py
swift/common/direct_client.py
swift/common/exceptions.py
swift/common/http.py
swift/common/internal_client.py
swift/common/manager.py
swift/common/memcached.py
swift/common/request_helpers.py
swift/common/swob.py
swift/common/utils.py
swift/common/wsgi.py
swift/common/middleware/__init__.py
swift/common/middleware/account_quotas.py
swift/common/middleware/acl.py
swift/common/middleware/bulk.py
swift/common/middleware/catch_errors.py
swift/common/middleware/cname_lookup.py
swift/common/middleware/container_quotas.py
swift/common/middleware/container_sync.py
swift/common/middleware/crossdomain.py
swift/common/middleware/dlo.py
swift/common/middleware/domain_remap.py
swift/common/middleware/formpost.py
swift/common/middleware/gatekeeper.py
swift/common/middleware/healthcheck.py
swift/common/middleware/keystoneauth.py
swift/common/middleware/list_endpoints.py
swift/common/middleware/memcache.py
swift/common/middleware/name_check.py
swift/common/middleware/proxy_logging.py
swift/common/middleware/ratelimit.py
swift/common/middleware/recon.py
swift/common/middleware/slo.py
swift/common/middleware/staticweb.py
swift/common/middleware/tempauth.py
swift/common/middleware/tempurl.py
swift/common/ring/__init__.py
swift/common/ring/builder.py
swift/common/ring/ring.py
swift/common/ring/utils.py
swift/container/__init__.py
swift/container/auditor.py
swift/container/backend.py
swift/container/replicator.py
swift/container/server.py
swift/container/sync.py
swift/container/updater.py
swift/obj/__init__.py
swift/obj/auditor.py
swift/obj/diskfile.py
swift/obj/expirer.py
swift/obj/mem_diskfile.py
swift/obj/mem_server.py
swift/obj/replicator.py
swift/obj/server.py
swift/obj/ssync_receiver.py
swift/obj/ssync_sender.py
swift/obj/updater.py
swift/proxy/__init__.py
swift/proxy/server.py
swift/proxy/controllers/__init__.py
swift/proxy/controllers/account.py
swift/proxy/controllers/base.py
swift/proxy/controllers/container.py
swift/proxy/controllers/info.py
swift/proxy/controllers/obj.py
test/__init__.py
test/sample.conf
test/functional/__init__.py
test/functional/swift_test_client.py
test/functional/swift_testing.py
test/functional/test_account.py
test/functional/test_container.py
test/functional/test_object.py
test/functional/tests.py
test/probe/__init__.py
test/probe/common.py
test/probe/test_account_failures.py
test/probe/test_account_get_fake_responses_match.py
test/probe/test_container_failures.py
test/probe/test_empty_device_handoff.py
test/probe/test_object_async_update.py
test/probe/test_object_failures.py
test/probe/test_object_handoff.py
test/probe/test_replication_servers_working.py
test/unit/__init__.py
test/unit/account/__init__.py
test/unit/account/test_auditor.py
test/unit/account/test_backend.py
test/unit/account/test_reaper.py
test/unit/account/test_replicator.py
test/unit/account/test_server.py
test/unit/cli/__init__.py
test/unit/cli/test_info.py
test/unit/cli/test_recon.py
test/unit/cli/test_ringbuilder.py
test/unit/common/__init__.py
test/unit/common/corrupted_example.db
test/unit/common/malformed_example.db
test/unit/common/test_bufferedhttp.py
test/unit/common/test_constraints.py
test/unit/common/test_container_sync_realms.py
test/unit/common/test_daemon.py
test/unit/common/test_db.py
test/unit/common/test_db_replicator.py
test/unit/common/test_direct_client.py
test/unit/common/test_exceptions.py
test/unit/common/test_internal_client.py
test/unit/common/test_manager.py
test/unit/common/test_memcached.py
test/unit/common/test_request_helpers.py
test/unit/common/test_swob.py
test/unit/common/test_utils.py
test/unit/common/test_wsgi.py
test/unit/common/middleware/__init__.py
test/unit/common/middleware/helpers.py
test/unit/common/middleware/test_account_quotas.py
test/unit/common/middleware/test_acl.py
test/unit/common/middleware/test_bulk.py
test/unit/common/middleware/test_cname_lookup.py
test/unit/common/middleware/test_container_sync.py
test/unit/common/middleware/test_crossdomain.py
test/unit/common/middleware/test_dlo.py
test/unit/common/middleware/test_domain_remap.py
test/unit/common/middleware/test_except.py
test/unit/common/middleware/test_formpost.py
test/unit/common/middleware/test_gatekeeper.py
test/unit/common/middleware/test_healthcheck.py
test/unit/common/middleware/test_keystoneauth.py
test/unit/common/middleware/test_list_endpoints.py
test/unit/common/middleware/test_memcache.py
test/unit/common/middleware/test_name_check.py
test/unit/common/middleware/test_proxy_logging.py
test/unit/common/middleware/test_quotas.py
test/unit/common/middleware/test_ratelimit.py
test/unit/common/middleware/test_recon.py
test/unit/common/middleware/test_slo.py
test/unit/common/middleware/test_staticweb.py
test/unit/common/middleware/test_tempauth.py
test/unit/common/middleware/test_tempurl.py
test/unit/common/ring/__init__.py
test/unit/common/ring/test_builder.py
test/unit/common/ring/test_ring.py
test/unit/common/ring/test_utils.py
test/unit/container/__init__.py
test/unit/container/test_auditor.py
test/unit/container/test_backend.py
test/unit/container/test_replicator.py
test/unit/container/test_server.py
test/unit/container/test_sync.py
test/unit/container/test_updater.py
test/unit/obj/__init__.py
test/unit/obj/test_auditor.py
test/unit/obj/test_diskfile.py
test/unit/obj/test_expirer.py
test/unit/obj/test_replicator.py
test/unit/obj/test_server.py
test/unit/obj/test_ssync_receiver.py
test/unit/obj/test_ssync_sender.py
test/unit/obj/test_updater.py
test/unit/proxy/__init__.py
test/unit/proxy/test_mem_server.py
test/unit/proxy/test_server.py
test/unit/proxy/controllers/__init__.py
test/unit/proxy/controllers/test_account.py
test/unit/proxy/controllers/test_base.py
test/unit/proxy/controllers/test_container.py
test/unit/proxy/controllers/test_info.py
test/unit/proxy/controllers/test_obj.py
test/unit/test_locale/README
test/unit/test_locale/__init__.py
test/unit/test_locale/eo.po
test/unit/test_locale/messages.mo
test/unit/test_locale/test_locale.py
test/unit/test_locale/eo/LC_MESSAGES/swift.moswift-1.13.1/LICENSE0000664000175400017540000002613612323703611015102 0ustar  jenkinsjenkins00000000000000
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/

   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

   1. Definitions.

      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.

      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.

      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.

      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.

      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.

      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.

      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).

      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.

      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."

      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.

   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.

   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.

   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:

      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and

      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and

      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and

      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.

      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.

   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.

   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.

   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.

   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.

   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.

   END OF TERMS AND CONDITIONS

   APPENDIX: How to apply the Apache License to your work.

      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "[]"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

   Copyright [yyyy] [name of copyright owner]

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
swift-1.13.1/.probetests0000775000175400017540000000024712323703611016266 0ustar  jenkinsjenkins00000000000000#!/bin/bash

SRC_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))")

cd ${SRC_DIR}/test/probe
nosetests --exe $@
rvalue=$?
cd -

exit $rvalue
swift-1.13.1/babel.cfg0000664000175400017540000000002112323703611015604 0ustar  jenkinsjenkins00000000000000[python: **.py]

swift-1.13.1/requirements.txt0000664000175400017540000000016112323703611017347 0ustar  jenkinsjenkins00000000000000dnspython>=1.9.4
eventlet>=0.9.15
greenlet>=0.3.1
netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
swift-1.13.1/MANIFEST.in0000664000175400017540000000047112323703611015625 0ustar  jenkinsjenkins00000000000000include AUTHORS LICENSE .functests .unittests .probetests test/__init__.py
include CHANGELOG CONTRIBUTING.md README.md
include babel.cfg
include test/sample.conf
include tox.ini
include requirements.txt test-requirements.txt
graft doc
graft etc
graft locale
graft test/functional
graft test/probe
graft test/unit
swift-1.13.1/AUTHORS0000664000175400017540000001356512323703611015147 0ustar  jenkinsjenkins00000000000000Maintainer
----------
OpenStack Foundation
IRC: #openstack on irc.freenode.net

Original Authors
----------------
Michael Barton (mike@weirdlooking.com)
John Dickinson (me@not.mn)
Greg Holt (gholt@rackspace.com)
Greg Lange (greglange@gmail.com)
Jay Payne (letterj@gmail.com)
Will Reese (wreese@gmail.com)
Chuck Thier (cthier@gmail.com)

Contributors
------------
Mehdi Abaakouk (mehdi.abaakouk@enovance.com)
Jesse Andrews (anotherjesse@gmail.com)
Joe Arnold (joe@swiftstack.com)
Ionuț Arțăriși (iartarisi@suse.cz)
Luis de Bethencourt (luis@debethencourt.com)
Darrell Bishop (darrell@swiftstack.com)
James E. Blair (jeblair@openstack.org)
Fabien Boucher (fabien.boucher@enovance.com)
Chmouel Boudjnah (chmouel@enovance.com)
Clark Boylan (clark.boylan@gmail.com)
Pádraig Brady (pbrady@redhat.com)
Russell Bryant (rbryant@redhat.com)
Brian D. Burns (iosctr@gmail.com)
Devin Carlen (devin.carlen@gmail.com)
Thierry Carrez (thierry@openstack.org)
Zap Chang (zapchang@gmail.com)
François Charlier (francois.charlier@enovance.com)
Ray Chen (oldsharp@163.com)
Brian Cline (bcline@softlayer.com)
Alistair Coles (alistair.coles@hp.com)
Brian Curtin (brian.curtin@rackspace.com)
Julien Danjou (julien@danjou.info)
Ksenia Demina (kdemina@mirantis.com)
Dan Dillinger (dan.dillinger@sonian.net)
Morgan Fainberg (m@metacloud.com)
ZhiQiang Fan (aji.zqfan@gmail.com)
Flaper Fesp (flaper87@gmail.com)
Tom Fifield (tom@openstack.org)
Florent Flament (florent.flament-ext@cloudwatt.com)
Gaurav B. Gangalwar (gaurav@gluster.com)
Alex Gaynor (alex.gaynor@gmail.com)
Anne Gentle (anne@openstack.org)
Clay Gerrard (clay.gerrard@gmail.com)
Mark Gius (launchpad@markgius.com)
David Goetz (david.goetz@rackspace.com)
Jonathan Gonzalez V (jonathan.abdiel@gmail.com)
Joe Gordon (jogo@cloudscaling.com)
David Hadas (davidh@il.ibm.com)
Soren Hansen (soren@linux2go.dk)
Richard (Rick) Hawkins (richard.hawkins@rackspace.com)
Doug Hellmann (doug.hellmann@dreamhost.com)
Dan Hersam (dan.hersam@hp.com)
Derek Higgins (derekh@redhat.com)
Florian Hines (syn@ronin.io)
Alex Holden (alex@alexjonasholden.com)
Edward Hope-Morley (opentastic@gmail.com)
Kun Huang (gareth@unitedstack.com)
Matthieu Huin (mhu@enovance.com)
Hodong Hwang (hodong.hwang@kt.com)
Motonobu Ichimura (motonobu@gmail.com)
Shri Javadekar (shrinand@maginatics.com)
Iryoung Jeong (iryoung@gmail.com)
Paul Jimenez (pj@place.org)
Zhang Jinnan (ben.os@99cloud.net)
Jason Johnson (jajohnson@softlayer.com)
Brian K. Jones (bkjones@gmail.com)
Kiyoung Jung (kiyoung.jung@kt.com)
Matt Kassawara (mkassawara@gmail.com)
Morita Kazutaka (morita.kazutaka@gmail.com)
Josh Kearney (josh@jk0.org)
Ilya Kharin (ikharin@mirantis.com)
Dae S. Kim (dae@velatum.com)
Eugene Kirpichov (ekirpichov@gmail.com)
Leah Klearman (lklrmn@gmail.com)
Steve Kowalik (steven@wedontsleep.org)
Sergey Kraynev (skraynev@mirantis.com)
Sushil Kumar (sushil.kumar2@globallogic.com)
Madhuri Kumari (madhuri.rai07@gmail.com)
Steven Lang (Steven.Lang@hgst.com)
Gonéri Le Bouder (goneri.lebouder@enovance.com)
Ed Leafe (ed.leafe@rackspace.com)
Thomas Leaman (thomas.leaman@hp.com)
Eohyung Lee (liquid@kt.com)
Jamie Lennox (jlennox@redhat.com)
Tong Li (litong01@us.ibm.com)
Changbin Liu (changbin.liu@gmail.com)
Victor Lowther (victor.lowther@gmail.com)
Christopher MacGown (chris@pistoncloud.com)
Sergey Lukjanov (slukjanov@mirantis.com)
Zhongyue Luo (zhongyue.nah@intel.com)
Dragos Manolescu (dragosm@hp.com)
Juan J. Martinez (juan@memset.com)
Marcelo Martins (btorch@gmail.com)
Donagh McCabe (donagh.mccabe@hp.com)
Andy McCrae (andy.mccrae@gmail.com)
Paul McMillan (paul.mcmillan@nebula.com)
Ewan Mellor (ewan.mellor@citrix.com)
Samuel Merritt (sam@swiftstack.com)
Stephen Milton (milton@isomedia.com)
Jola Mirecka (jola.mirecka@hp.com)
Dirk Mueller (dirk@dmllr.de)
Russ Nelson (russ@crynwr.com)
Maru Newby (mnewby@internap.com)
Newptone (xingchao@unitedstack.com)
Colin Nicholson (colin.nicholson@iomart.com)
Zhenguo Niu (zhenguo@unitedstack.com)
Eamonn O'Toole (eamonn.otoole@hp.com)
Prashanth Pai (ppai@redhat.com)
Sascha Peilicke (saschpe@gmx.de)
Constantine Peresypkin (constantine.peresypk@rackspace.com)
Dieter Plaetinck (dieter@vimeo.com)
Peter Portante (peter.portante@redhat.com)
Dan Prince (dprince@redhat.com)
Felipe Reyes (freyes@tty.cl)
Li Riqiang (lrqrun@gmail.com)
Victor Rodionov (victor.rodionov@nexenta.com)
Aaron Rosen (arosen@nicira.com)
Brent Roskos (broskos@internap.com)
Cristian A Sanchez (cristian.a.sanchez@intel.com)
Christian Schwede (info@cschwede.de)
Andrew Clay Shafer (acs@parvuscaptus.com)
Chuck Short (chuck.short@canonical.com)
Michael Shuler (mshuler@gmail.com)
David Moreau Simard (dmsimard@iweb.com)
Scott Simpson (sasimpson@gmail.com)
Liu Siqi (meizu647@gmail.com)
Adrian Smith (adrian_f_smith@dell.com)
Jon Snitow (otherjon@swiftstack.com)
TheSriram (sriram@klusterkloud.com)
Jeremy Stanley (fungi@yuggoth.org)
Tobias Stevenson (tstevenson@vbridges.com)
Yuriy Taraday (yorik.sar@gmail.com)
Monty Taylor (mordred@inaugust.com)
Caleb Tennis (caleb.tennis@gmail.com)
Rainer Toebbicke (Rainer.Toebbicke@cern.ch)
Fujita Tomonori (fujita.tomonori@lab.ntt.co.jp)
Kapil Thangavelu (kapil.foss@gmail.com)
Dean Troyer (dtroyer@gmail.com)
Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp)
Dmitry Ukov (dukov@mirantis.com)
Vincent Untz (vuntz@suse.com)
Daniele Valeriani (daniele@dvaleriani.net)
Koert van der Veer (koert@cloudvps.com)
Vladimir Vechkanov (vvechkanov@mirantis.com)
Hou Ming Wang (houming.wang@easystack.cn)
Shane Wang (shane.wang@intel.com)
Yaguang Wang (yaguang.wang@intel.com)
Chris Wedgwood (cw@f00f.org)
Conrad Weidenkeller (conrad.weidenkeller@rackspace.com)
Doug Weimer (dweimer@gmail.com)
Wu Wenxiang (wu.wenxiang@99cloud.net)
Cory Wright (cory.wright@rackspace.com)
Ye Jia Xu (xyj.asmy@gmail.com)
Alex Yang (alex890714@gmail.com)
Yee (mail.zhang.yee@gmail.com)
Guang Yee (guang.yee@hp.com)
Pete Zaitcev (zaitcev@kotori.zaitcev.us)
niu-zglinux (Niu.ZGlinux@gmail.com)
Jian Zhang (jian.zhang@intel.com)
Ning Zhang (ning@zmanda.com)
Yuan Zhou (yuan.zhou@intel.com)
swift-1.13.1/tox.ini0000664000175400017540000000245012323703611015401 0ustar  jenkinsjenkins00000000000000[tox]
envlist = py26,py27,pep8
minversion = 1.6
skipsdist = True

[testenv]
usedevelop = True
install_command = pip install --allow-external netifaces --allow-insecure netifaces -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
         NOSE_WITH_OPENSTACK=1
         NOSE_OPENSTACK_COLOR=1
         NOSE_OPENSTACK_RED=0.05
         NOSE_OPENSTACK_YELLOW=0.025
         NOSE_OPENSTACK_SHOW_ELAPSED=1
         NOSE_OPENSTACK_STDOUT=1
         NOSE_WITH_COVERAGE=1
         NOSE_COVER_BRANCHES=1
deps =
  -r{toxinidir}/requirements.txt
  -r{toxinidir}/test-requirements.txt
commands = nosetests {posargs:test/unit}

[testenv:cover]
setenv = VIRTUAL_ENV={envdir}
         NOSE_WITH_COVERAGE=1
         NOSE_COVER_BRANCHES=1
         NOSE_COVER_HTML=1
         NOSE_COVER_HTML_DIR={toxinidir}/cover

[tox:jenkins]
downloadcache = ~/cache/pip

[testenv:pep8]
commands =
  flake8 swift test doc setup.py
  flake8 --filename=swift* bin

[testenv:venv]
commands = {posargs}

[flake8]
# it's not a bug that we aren't using all of hacking
# H102 -> apache2 license exists
# H103 -> license is apache
# H201 -> no bare excepts # add when hacking supports noqa
# H501 -> don't use locals() for str formatting
# H903 -> \n not \r\n
ignore = H
select = F,E,W,H102,H103,H501,H903
exclude = .venv,.tox,dist,doc,*egg
show-source = True