diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index f0b629a2..a21b5b82 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -224,6 +224,13 @@ def https(): return True if config_get('ssl_cert') and config_get('ssl_key'): return True + # Local import to avoid ciruclar dependency. + import charmhelpers.contrib.openstack.cert_utils as cert_utils + if ( + cert_utils.get_certificate_request() and not + cert_utils.get_requests_for_local_unit("certificates") + ): + return False for r_id in relation_ids('certificates'): for unit in relation_list(r_id): ca = relation_get('ca', rid=r_id, unit=unit) @@ -327,7 +334,7 @@ def valid_hacluster_config(): ''' vip = config_get('vip') dns = config_get('dns-ha') - if not(bool(vip) ^ bool(dns)): + if not (bool(vip) ^ bool(dns)): msg = ('HA: Either vip or dns-ha must be set but not both in order to ' 'use high availability') status_set('blocked', msg) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index b356d64c..12738d54 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -477,7 +477,7 @@ def ns_query(address): try: answers = dns.resolver.query(address, rtype) - except dns.resolver.NXDOMAIN: + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): return None if answers: @@ -552,7 +552,7 @@ def port_has_listener(address, port): """ cmd = ['nc', '-z', address, str(port)] result = subprocess.call(cmd) - return not(bool(result)) + return not (bool(result)) def assert_charm_supports_ipv6(): diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index d9f004a7..8da93105 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -652,7 +652,7 @@ def patch_ports_on_bridge(bridge): uuid_for_port( interface['options']['peer'])), interface['options']['peer']) - yield(Patch(this_end, other_end)) + yield Patch(this_end, other_end) # We expect one result and it is ok if it turns out to be a port # for a different bridge. However we need a break here to satisfy # the for/else check which is in place to detect interface referring diff --git a/hooks/charmhelpers/contrib/network/ovs/ovsdb.py b/hooks/charmhelpers/contrib/network/ovs/ovsdb.py index 2f1e53da..37f6f6f8 100644 --- a/hooks/charmhelpers/contrib/network/ovs/ovsdb.py +++ b/hooks/charmhelpers/contrib/network/ovs/ovsdb.py @@ -199,7 +199,7 @@ def _deserialize_ovsdb(self, data): decoded_set = [] for el in data[1]: decoded_set.append(self._deserialize_ovsdb(el)) - return(decoded_set) + return decoded_set # fall back to normal processing below break diff --git a/hooks/charmhelpers/contrib/openstack/cert_utils.py b/hooks/charmhelpers/contrib/openstack/cert_utils.py index 5c961c58..6620f59f 100644 --- a/hooks/charmhelpers/contrib/openstack/cert_utils.py +++ b/hooks/charmhelpers/contrib/openstack/cert_utils.py @@ -409,13 +409,33 @@ def get_requests_for_local_unit(relation_name=None): relation_name = relation_name or 'certificates' bundles = [] for rid in relation_ids(relation_name): + sent = relation_get(rid=rid, unit=local_unit()) + legacy_keys = ['certificate_name', 'common_name'] + is_legacy_request = set(sent).intersection(legacy_keys) for unit in related_units(rid): data = relation_get(rid=rid, unit=unit) - if data.get(raw_certs_key): - bundles.append({ - 'ca': data['ca'], - 'chain': data.get('chain'), - 'certs': json.loads(data[raw_certs_key])}) + # Note: Bug#2028683 - data may not be available if the certificates + # relation hasn't been populated by the providing charm. If no 'ca' + # in the data then don't attempt the bundle at all. + if data.get('ca'): + if data.get(raw_certs_key): + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': json.loads(data[raw_certs_key]) + }) + elif is_legacy_request: + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': { + sent['common_name']: { + 'cert': data.get(local_name + '.server.cert'), + 'key': data.get(local_name + '.server.key') + } + } + }) + return bundles diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index df00c970..2c21ed35 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -463,6 +463,7 @@ def __call__(self): int_host = format_ipv6_addr(int_host) or int_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' + admin_role = rdata.get('admin_role') or 'Admin' int_protocol = rdata.get('internal_protocol') or 'http' api_version = rdata.get('api_version') or '2.0' ctxt.update({'service_port': rdata.get('service_port'), @@ -474,6 +475,7 @@ def __call__(self): 'admin_tenant_name': rdata.get('service_tenant'), 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), + 'admin_role': admin_role, 'service_protocol': svc_protocol, 'auth_protocol': auth_protocol, 'internal_protocol': int_protocol, diff --git a/hooks/charmhelpers/contrib/openstack/ssh_migrations.py b/hooks/charmhelpers/contrib/openstack/ssh_migrations.py index 96b9f71d..0512e3a5 100644 --- a/hooks/charmhelpers/contrib/openstack/ssh_migrations.py +++ b/hooks/charmhelpers/contrib/openstack/ssh_migrations.py @@ -310,7 +310,7 @@ def ssh_known_hosts_lines(application_name, user=None): for hosts_line in hosts: if hosts_line.rstrip(): known_hosts_list.append(hosts_line.rstrip()) - return(known_hosts_list) + return known_hosts_list def ssh_authorized_keys_lines(application_name, user=None): @@ -327,7 +327,7 @@ def ssh_authorized_keys_lines(application_name, user=None): for authkey_line in keys: if authkey_line.rstrip(): authorized_keys_list.append(authkey_line.rstrip()) - return(authorized_keys_list) + return authorized_keys_list def ssh_compute_remove(public_key, application_name, user=None): diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 875e1393..626ecbab 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -82,7 +82,11 @@ backend {{ service }}_{{ frontend }} {% endif -%} {% endif -%} {% for unit, address in frontends[frontend]['backends'].items() -%} + {% if https -%} + server {{ unit }} {{ address }}:{{ ports[1] }} check check-ssl verify none + {% else -%} server {{ unit }} {{ address }}:{{ ports[1] }} check + {% endif -%} {% endfor %} {% endfor -%} {% endfor -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken index c9b01528..dbad506f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -12,4 +12,6 @@ signing_dir = {{ signing_dir }} {% if service_type -%} service_type = {{ service_type }} {% endif -%} +service_token_roles = {{ admin_role }} +service_token_roles_required = True {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 14c25b4d..139a0512 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -22,4 +22,6 @@ signing_dir = {{ signing_dir }} {% if use_memcache == true %} memcached_servers = {{ memcache_url }} {% endif -%} +service_token_roles = {{ admin_role }} +service_token_roles_required = True {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-service-user b/hooks/charmhelpers/contrib/openstack/templates/section-service-user new file mode 100644 index 00000000..ff454086 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-service-user @@ -0,0 +1,11 @@ +{% if auth_host -%} +[service_user] +send_service_user_token = true +auth_type = password +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +project_domain_name = service_domain +user_domain_name = service_domain +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d5d301e6..b6171fc0 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1039,7 +1039,7 @@ def _determine_os_workload_status( state, message, lambda: charm_func(configs)) if state is None: - state, message = _ows_check_services_running(services, ports) + state, message = ows_check_services_running(services, ports) if state is None: state = 'active' @@ -1213,7 +1213,12 @@ def _ows_check_charm_func(state, message, charm_func_with_configs): return state, message +@deprecate("use ows_check_services_running() instead", "2022-05", log=juju_log) def _ows_check_services_running(services, ports): + return ows_check_services_running(services, ports) + + +def ows_check_services_running(services, ports): """Check that the services that should be running are actually running and that any ports specified are being listened to. @@ -1320,7 +1325,7 @@ def _check_listening_on_services_ports(services, test=False): @param test: default=False, if False, test for closed, otherwise open. @returns OrderedDict(service: [port-not-open, ...]...), [boolean] """ - test = not(not(test)) # ensure test is True or False + test = not (not (test)) # ensure test is True or False all_ports = list(itertools.chain(*services.values())) ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports] map_ports = OrderedDict() @@ -1544,7 +1549,7 @@ def is_unit_paused_set(): with unitdata.HookData()() as t: kv = t[0] # transform something truth-y into a Boolean. - return not(not(kv.get('unit-paused'))) + return not (not (kv.get('unit-paused'))) except Exception: return False @@ -2143,7 +2148,7 @@ def is_unit_upgrading_set(): with unitdata.HookData()() as t: kv = t[0] # transform something truth-y into a Boolean. - return not(not(kv.get('unit-upgrading'))) + return not (not (kv.get('unit-upgrading'))) except Exception: return False @@ -2599,6 +2604,23 @@ def get_subordinate_release_packages(os_release, package_type='deb'): return SubordinatePackages(install, purge) +def get_subordinate_services(): + """Iterate over subordinate relations and get service information. + + In a similar fashion as with get_subordinate_release_packages(), + principle charms can retrieve a list of services advertised by their + subordinate charms. This is useful to know about subordinate services when + pausing, resuming or upgrading a principle unit. + + :returns: Name of all services advertised by all subordinates + :rtype: Set[str] + """ + services = set() + for rdata in container_scoped_relation_get('services'): + services |= set(json.loads(rdata or '[]')) + return services + + os_restart_on_change = partial( pausable_restart_on_change, can_restart_now_f=deferred_events.check_and_record_restart_request, diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 994ec8a0..773aac37 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -926,7 +926,7 @@ def pwgen(length=None): random_generator = random.SystemRandom() random_chars = [ random_generator.choice(alphanumeric_chars) for _ in range(length)] - return(''.join(random_chars)) + return ''.join(random_chars) def is_phy_iface(interface): diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py index 36d6bce9..7ab7ce3e 100644 --- a/hooks/charmhelpers/fetch/snap.py +++ b/hooks/charmhelpers/fetch/snap.py @@ -52,7 +52,7 @@ def _snap_exec(commands): :param commands: List commands :return: Integer exit code """ - assert type(commands) == list + assert isinstance(commands, list) retry_count = 0 return_code = None diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 6c7cf6fc..ab0357e0 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -224,6 +224,10 @@ 'yoga/proposed': 'focal-proposed/yoga', 'focal-yoga/proposed': 'focal-proposed/yoga', 'focal-proposed/yoga': 'focal-proposed/yoga', + + # OVN + 'focal-ovn-22.03': 'focal-updates/ovn-22.03', + 'focal-ovn-22.03/proposed': 'focal-proposed/ovn-22.03', } @@ -683,6 +687,7 @@ def add_source(source, key=None, fail_invalid=False): (r"^cloud-archive:(.*)$", _add_apt_repository), (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), + (r"^cloud:(.*)-(ovn-.*)$", _add_cloud_distro_check), (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), (r"^cloud:(.*)$", _add_cloud_pocket), (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), @@ -746,6 +751,11 @@ def _add_apt_repository(spec): ) +def __write_sources_list_d_actual_pocket(file, actual_pocket): + with open('/etc/apt/sources.list.d/{}'.format(file), 'w') as apt: + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + + def _add_cloud_pocket(pocket): """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list @@ -765,8 +775,9 @@ def _add_cloud_pocket(pocket): 'Unsupported cloud: source option %s' % pocket) actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: - apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + __write_sources_list_d_actual_pocket( + 'cloud-archive{}.list'.format('' if 'ovn' not in pocket else '-ovn'), + actual_pocket) def _add_cloud_staging(cloud_archive_release, openstack_release): diff --git a/test-requirements.txt b/test-requirements.txt index e01c8854..4b78a270 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,6 +10,7 @@ pyparsing<3.0.0 # aodhclient is pinned in zaza and needs pyparsing < 3.0.0, but cffi also needs it, so pin here. cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85 +testtools<2.6.0 requests>=2.18.4 diff --git a/tox.ini b/tox.ini index c41077de..6e959cf4 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ skip_missing_interpreters = False # lead to fetching the latest pip in the func* tox targets, see # https://stackoverflow.com/a/38133283 requires = + tox < 4.0.0 pip < 20.3 virtualenv < 20.0 setuptools < 50.0.0 @@ -83,7 +84,8 @@ commands = stestr run --slowest {posargs} [testenv:pep8] basepython = python3 deps = flake8==3.9.2 - charm-tools==2.8.3 + PyYAML==6.0.1 + charm-tools==2.8.6 commands = flake8 {posargs} hooks unit_tests tests actions lib files charm-proof diff --git a/unit_tests/test_neutron_api_hooks.py b/unit_tests/test_neutron_api_hooks.py index ffd4b674..04450561 100644 --- a/unit_tests/test_neutron_api_hooks.py +++ b/unit_tests/test_neutron_api_hooks.py @@ -161,6 +161,7 @@ def test_install_hook(self, mock_maybe_set_os_install_release): _port_calls = [call(port) for port in _ports] self.determine_packages.return_value = _pkgs self.determine_ports.return_value = _ports + self.patch('additional_install_locations') self._call_hook('install') self.configure_installation_source.assert_called_with( 'distro' @@ -183,6 +184,8 @@ def test_nuage_install_hook(self): _port_calls = [call(port) for port in _ports] self.determine_packages.return_value = _pkgs self.determine_ports.return_value = _ports + self.patch('additional_install_locations') + self.patch('maybe_set_os_install_release') self._call_hook('install') self.configure_installation_source.assert_called_with( 'distro' @@ -201,6 +204,7 @@ def test_config_changed(self, conf_https): self.dvr_router_present.return_value = False self.l3ha_router_present.return_value = False self.relation_ids.side_effect = self._fake_relids + self.patch('additional_install_locations') _n_api_rel_joined = self.patch('neutron_api_relation_joined') _n_plugin_api_rel_joined =\ self.patch('neutron_plugin_api_relation_joined') @@ -241,6 +245,7 @@ def test_config_changed_with_openstack_upgrade_action(self): self.remove_old_packages.return_value = False self.openstack_upgrade_available.return_value = True self.test_config.set('action-managed-upgrade', True) + self.patch('additional_install_locations') self._call_hook('config-changed') @@ -250,6 +255,7 @@ def test_config_changed_with_purge(self): self.remove_old_packages.return_value = True self.services.return_value = ['neutron-server'] self.openstack_upgrade_available.return_value = False + self.patch('additional_install_locations') self._call_hook('config-changed') self.remove_old_packages.assert_called_once_with() self.service_restart.assert_called_once_with('neutron-server') @@ -267,14 +273,17 @@ def test_amqp_changed(self, plugin_joined): self.relation_ids.return_value = ['neutron-plugin-api-subordinate:1'] self.CONFIGS.complete_contexts.return_value = ['amqp'] self._call_hook('amqp-relation-changed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) self.relation_ids.assert_called_with('neutron-plugin-api-subordinate') plugin_joined.assert_called_with( relid='neutron-plugin-api-subordinate:1') def test_amqp_departed(self): + self.CONFIGS.complete_contexts.return_value = { + 'amqp': None + } self._call_hook('amqp-relation-departed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) def test_db_joined(self): self.get_relation_ip.return_value = '10.0.0.1' @@ -395,7 +404,7 @@ def test_identity_changed(self, conf_https, mock_manage_plugin): _api_rel_joined = self.patch('neutron_api_relation_joined') self.relation_ids.side_effect = self._fake_relids self._call_hook('identity-service-relation-changed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) self.assertTrue(_api_rel_joined.called) @patch.object(hooks, 'canonical_url') @@ -464,7 +473,7 @@ def test_vsd_api_relation_changed(self): }) self._call_hook('vsd-rest-api-relation-changed') config_file = '/etc/neutron/plugins/nuage/nuage_plugin.ini' - self.assertTrue(self.CONFIGS.write.called_with(config_file)) + self.CONFIGS.write.asseert_called_with(config_file) def test_vsd_api_relation_joined(self): self.os_release.return_value = 'kilo' @@ -488,12 +497,12 @@ def test_vsd_api_relation_joined(self): def test_neutron_api_relation_changed(self): self.CONFIGS.complete_contexts.return_value = ['shared-db'] self._call_hook('neutron-api-relation-changed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) def test_neutron_api_relation_changed_incomplere_ctxt(self): self.CONFIGS.complete_contexts.return_value = [] self._call_hook('neutron-api-relation-changed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) @patch.object(hooks, 'is_api_ready') def test_neutron_load_balancer_relation_joined(self, is_api_ready): @@ -1023,11 +1032,11 @@ def test_designate_peer_joined(self): }) self.relation_ids.side_effect = self._fake_relids self._call_hook('external-dns-relation-joined') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write_all.assert_called_with() def test_designate_peer_departed(self): self._call_hook('external-dns-relation-departed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write_all.assert_called_with() def test_infoblox_peer_changed(self): self.is_db_initialised.return_value = True @@ -1036,12 +1045,15 @@ def test_infoblox_peer_changed(self): }) self.os_release.return_value = 'queens' self.relation_ids.side_effect = self._fake_relids + self.CONFIGS.complete_contexts.return_value = { + 'infoblox-neutron': None + } self._call_hook('infoblox-neutron-relation-changed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write.assert_called_with(NEUTRON_CONF) def test_infoblox_peer_departed(self): self._call_hook('infoblox-neutron-relation-departed') - self.assertTrue(self.CONFIGS.write.called_with(NEUTRON_CONF)) + self.CONFIGS.write_all.assert_called_with() @patch.object(hooks, 'NeutronApiSDNContext') @patch.object(hooks, 'NeutronCCContext')