diff options
22 files changed, 272 insertions, 232 deletions
@@ -15,15 +15,33 @@ PYTHON := /usr/bin/env python -unittest: - tests/10-unit.test +clean: + rm -f .coverage + find . -name '*.pyc' -delete + rm -rf .venv + (which dh_clean && dh_clean) || true + +.venv: + sudo apt-get install -y gcc python-dev python-virtualenv python-apt + virtualenv .venv --system-site-packages + .venv/bin/pip install -I -r test_requirements.txt + +lint: .venv + .venv/bin/flake8 --exclude hooks/charmhelpers hooks tests unit_tests + +test: .venv + @echo Starting unit tests... + .venv/bin/nosetests -s --nologcapture --with-coverage $(EXTRA) unit_tests/ + +functional_test: + @echo Starting amulet tests... + @juju test -v -p AMULET_HTTP_PROXY --timeout 900 sync: @mkdir -p bin - @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > bin/charm_helpers_sync.py + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > bin/charm_helpers_sync.py @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml -clean: - @find . -name \*.pyc -delete - @find . -name '*.bak' -delete - +publish: lint unit_test + bzr push lp:charms/mongodb + bzr push lp:charms/trusty/mongodb diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 7e24965..7a71714 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -3,4 +3,4 @@ destination: hooks/charmhelpers include: - core - fetch - - contrib.hahelpers.cluster \ No newline at end of file + - contrib.hahelpers.cluster diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 6d97200..52ce4b7 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -13,9 +13,10 @@ clustering-related helpers. import subprocess import os - from socket import gethostname as get_unit_hostname +import six + from charmhelpers.core.hookenv import ( log, relation_ids, @@ -77,7 +78,7 @@ def is_crm_leader(resource): "show", resource ] try: - status = subprocess.check_output(cmd) + status = subprocess.check_output(cmd).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -150,34 +151,42 @@ def https(): return False -def determine_api_port(public_port): +def determine_api_port(public_port, singlenode_mode=False): ''' Determine correct API server listening port based on existence of HTTPS reverse proxy and/or haproxy. public_port: int: standard public port for given service + singlenode_mode: boolean: Shuffle ports when only a single unit is present + returns: int: the correct listening port for the API service ''' i = 0 - if len(peer_units()) > 0 or is_clustered(): + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): i += 1 if https(): i += 1 return public_port - (i * 10) -def determine_apache_port(public_port): +def determine_apache_port(public_port, singlenode_mode=False): ''' Description: Determine correct apache listening port based on public IP + state of the cluster. public_port: int: standard public port for given service + singlenode_mode: boolean: Shuffle ports when only a single unit is present + returns: int: the correct listening port for the HAProxy service ''' i = 0 - if len(peer_units()) > 0 or is_clustered(): + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): i += 1 return public_port - (i * 10) @@ -197,7 +206,7 @@ def get_hacluster_config(): for setting in settings: conf[setting] = config_get(setting) missing = [] - [missing.append(s) for s, v in conf.iteritems() if v is None] + [missing.append(s) for s, v in six.iteritems(conf) if v is None] if missing: log('Insufficient config data to configure hacluster.', level=ERROR) raise HAIncompleteConfig diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index cfaf0a6..0adf0db 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -3,10 +3,11 @@ __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' +import io import os -class Fstab(file): +class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer for file `/etc/fstab` """ @@ -24,8 +25,8 @@ class Fstab(file): options = "defaults" self.options = options - self.d = d - self.p = p + self.d = int(d) + self.p = int(p) def __eq__(self, o): return str(self) == str(o) @@ -45,7 +46,7 @@ class Fstab(file): self._path = path else: self._path = self.DEFAULT_PATH - file.__init__(self, self._path, 'r+') + super(Fstab, self).__init__(self._path, 'rb+') def _hydrate_entry(self, line): # NOTE: use split with no arguments to split on any @@ -58,8 +59,9 @@ class Fstab(file): def entries(self): self.seek(0) for line in self.readlines(): + line = line.decode('us-ascii') try: - if not line.startswith("#"): + if line.strip() and not line.startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -75,14 +77,14 @@ class Fstab(file): if self.get_entry_by_attr('device', entry.device): return False - self.write(str(entry) + '\n') + self.write((str(entry) + '\n').encode('us-ascii')) self.truncate() return entry def remove_entry(self, entry): self.seek(0) - lines = self.readlines() + lines = [l.decode('us-ascii') for l in self.readlines()] found = False for index, line in enumerate(lines): @@ -97,7 +99,7 @@ class Fstab(file): lines.remove(line) self.seek(0) - self.write(''.join(lines)) + self.write(''.join(lines).encode('us-ascii')) self.truncate() return True diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 083a709..07d1f69 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -9,9 +9,14 @@ import json import yaml import subprocess import sys -import UserDict from subprocess import CalledProcessError +import six +if not six.PY3: + from UserDict import UserDict +else: + from collections import UserDict + CRITICAL = "CRITICAL" ERROR = "ERROR" WARNING = "WARNING" @@ -63,16 +68,18 @@ def log(message, level=None): command = ['juju-log'] if level: command += ['-l', level] + if not isinstance(message, six.string_types): + message = repr(message) command += [message] subprocess.call(command) -class Serializable(UserDict.IterableUserDict): +class Serializable(UserDict): """Wrapper, an object that can be serialized to yaml or json""" def __init__(self, obj): # wrap the object - UserDict.IterableUserDict.__init__(self) + UserDict.__init__(self) self.data = obj def __getattr__(self, attr): @@ -218,7 +225,7 @@ class Config(dict): prev_keys = [] if self._prev_dict is not None: prev_keys = self._prev_dict.keys() - return list(set(prev_keys + dict.keys(self))) + return list(set(prev_keys + list(dict.keys(self)))) def load_previous(self, path=None): """Load previous copy of config from disk. @@ -269,7 +276,7 @@ class Config(dict): """ if self._prev_dict: - for k, v in self._prev_dict.iteritems(): + for k, v in six.iteritems(self._prev_dict): if k not in self: self[k] = v with open(self.path, 'w') as f: @@ -284,7 +291,8 @@ def config(scope=None): config_cmd_line.append(scope) config_cmd_line.append('--format=json') try: - config_data = json.loads(subprocess.check_output(config_cmd_line)) + config_data = json.loads( + subprocess.check_output(config_cmd_line).decode('UTF-8')) if scope is not None: return config_data return Config(config_data) @@ -303,10 +311,10 @@ def relation_get(attribute=None, unit=None, rid=None): if unit: _args.append(unit) try: - return json.loads(subprocess.check_output(_args)) + return json.loads(subprocess.check_output(_args).decode('UTF-8')) except ValueError: return None - except CalledProcessError, e: + except CalledProcessError as e: if e.returncode == 2: return None raise @@ -318,7 +326,7 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs): relation_cmd_line = ['relation-set'] if relation_id is not None: relation_cmd_line.extend(('-r', relation_id)) - for k, v in (relation_settings.items() + kwargs.items()): + for k, v in (list(relation_settings.items()) + list(kwargs.items())): if v is None: relation_cmd_line.append('{}='.format(k)) else: @@ -335,7 +343,8 @@ def relation_ids(reltype=None): relid_cmd_line = ['relation-ids', '--format=json'] if reltype is not None: relid_cmd_line.append(reltype) - return json.loads(subprocess.check_output(relid_cmd_line)) or [] + return json.loads( + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] return [] @@ -346,7 +355,8 @@ def related_units(relid=None): units_cmd_line = ['relation-list', '--format=json'] if relid is not None: units_cmd_line.extend(('-r', relid)) - return json.loads(subprocess.check_output(units_cmd_line)) or [] + return json.loads( + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] @cached @@ -455,7 +465,7 @@ def unit_get(attribute): """Get the unit ID for the remote unit""" _args = ['unit-get', '--format=json', attribute] try: - return json.loads(subprocess.check_output(_args)) + return json.loads(subprocess.check_output(_args).decode('UTF-8')) except ValueError: return None diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 0b8bdc5..c6f1680 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -14,11 +14,12 @@ import string import subprocess import hashlib from contextlib import contextmanager - from collections import OrderedDict -from hookenv import log -from fstab import Fstab +import six + +from .hookenv import log +from .fstab import Fstab def service_start(service_name): @@ -54,7 +55,9 @@ def service(action, service_name): def service_running(service): """Determine whether a system service is running""" try: - output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT) + output = subprocess.check_output( + ['service', service, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError: return False else: @@ -67,7 +70,9 @@ def service_running(service): def service_available(service_name): """Determine whether a system service is available""" try: - subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) + subprocess.check_output( + ['service', service_name, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') except subprocess.CalledProcessError as e: return 'unrecognized service' not in e.output else: @@ -96,6 +101,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False): return user_info +def add_group(group_name, system_group=False): + """Add a group to the system""" + try: + group_info = grp.getgrnam(group_name) + log('group {0} already exists!'.format(group_name)) + except KeyError: + log('creating group {0}'.format(group_name)) + cmd = ['addgroup'] + if system_group: + cmd.append('--system') + else: + cmd.extend([ + '--group', + ]) + cmd.append(group_name) + subprocess.check_call(cmd) + group_info = grp.getgrnam(group_name) + return group_info + + def add_user_to_group(username, group): """Add a user to a group""" cmd = [ @@ -115,7 +140,7 @@ def rsync(from_path, to_path, flags='-r', options=None): cmd.append(from_path) cmd.append(to_path) log(" ".join(cmd)) - return subprocess.check_output(cmd).strip() + return subprocess.check_output(cmd).decode('UTF-8').strip() def symlink(source, destination): @@ -130,7 +155,7 @@ def symlink(source, destination): subprocess.check_call(cmd) -def mkdir(path, owner='root', group='root', perms=0555, force=False): +def mkdir(path, owner='root', group='root', perms=0o555, force=False): """Create a directory""" log("Making dir {} {}:{} {:o}".format(path, owner, group, perms)) @@ -146,7 +171,7 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False): os.chown(realpath, uid, gid) -def write_file(path, content, owner='root', group='root', perms=0444): +def write_file(path, content, owner='root', group='root', perms=0o444): """Create or overwrite a file with the contents of a string""" log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) uid = pwd.getpwnam(owner).pw_uid @@ -177,7 +202,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): cmd_args.extend([device, mountpoint]) try: subprocess.check_output(cmd_args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) return False @@ -191,7 +216,7 @@ def umount(mountpoint, persist=False): cmd_args = ['umount', mountpoint] try: subprocess.check_output(cmd_args) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: log('Error unmounting {}\n{}'.format(mountpoint, e.output)) return False @@ -218,8 +243,8 @@ def file_hash(path, hash_type='md5'): """ if os.path.exists(path): h = getattr(hashlib, hash_type)() - with open(path, 'r') as source: - h.update(source.read()) # IGNORE:E1101 - it does have update + with open(path, 'rb') as source: + h.update(source.read()) return h.hexdigest() else: return None @@ -297,7 +322,7 @@ def pwgen(length=None): if length is None: length = random.choice(range(35, 45)) alphanumeric_chars = [ - l for l in (string.letters + string.digits) + l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] random_chars = [ random.choice(alphanumeric_chars) for _ in range(length)] @@ -306,14 +331,14 @@ def pwgen(length=None): def list_nics(nic_type): '''Return a list of nics of given type(s)''' - if isinstance(nic_type, basestring): + if isinstance(nic_type, six.string_types): int_types = [nic_type] else: int_types = nic_type interfaces = [] for int_type in int_types: cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] - ip_output = subprocess.check_output(cmd).split('\n') + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): @@ -335,7 +360,7 @@ def set_nic_mtu(nic, mtu): def get_nic_mtu(nic): cmd = ['ip', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd).split('\n') + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') mtu = "" for line in ip_output: words = line.split() @@ -346,7 +371,7 @@ def get_nic_mtu(nic): def get_nic_hwaddr(nic): cmd = ['ip', '-o', '-0', 'addr', 'show', nic] - ip_output = subprocess.check_output(cmd) + ip_output = subprocess.check_output(cmd).decode('UTF-8') hwaddr = "" words = ip_output.split() if 'link/ether' in words: @@ -363,8 +388,8 @@ def cmp_pkgrevno(package, revno, pkgcache=None): ''' import apt_pkg - from charmhelpers.fetch import apt_cache if not pkgcache: + from charmhelpers.fetch import apt_cache pkgcache = apt_cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 7067b94..163a793 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -196,7 +196,7 @@ class StoredContext(dict): if not os.path.isabs(file_name): file_name = os.path.join(hookenv.charm_dir(), file_name) with open(file_name, 'w') as file_stream: - os.fchmod(file_stream.fileno(), 0600) + os.fchmod(file_stream.fileno(), 0o600) yaml.dump(config_data, file_stream) def read_context(self, file_name): @@ -211,15 +211,19 @@ class StoredContext(dict): class TemplateCallback(ManagerCallback): """ - Callback class that will render a Jinja2 template, for use as a ready action. + Callback class that will render a Jinja2 template, for use as a ready + action. + + :param str source: The template source file, relative to + `$CHARM_DIR/templates` - :param str source: The template source file, relative to `$CHARM_DIR/templates` :param str target: The target to write the rendered template to :param str owner: The owner of the rendered file :param str group: The group of the rendered file :param int perms: The permissions of the rendered file """ - def __init__(self, source, target, owner='root', group='root', perms=0444): + def __init__(self, source, target, + owner='root', group='root', perms=0o444): self.source = source self.target = target self.owner = owner diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py index 2c63885..83133fa 100644 --- a/hooks/charmhelpers/core/templating.py +++ b/hooks/charmhelpers/core/templating.py @@ -4,7 +4,8 @@ from charmhelpers.core import host from charmhelpers.core import hookenv -def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None): +def render(source, target, context, owner='root', group='root', + perms=0o444, templates_dir=None): """ Render a template. diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 2398e8e..0a126fc 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -5,10 +5,6 @@ from yaml import safe_load from charmhelpers.core.host import ( lsb_release ) -from urlparse import ( - urlparse, - urlunparse, -) import subprocess from charmhelpers.core.hookenv import ( config, @@ -16,6 +12,12 @@ from charmhelpers.core.hookenv import ( ) import os +import six +if six.PY3: + from urllib.parse import urlparse, urlunparse +else: + from urlparse import urlparse, urlunparse + CLOUD_ARCHIVE = """# Ubuntu Cloud Archive deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main @@ -149,7 +151,7 @@ def apt_install(packages, options=None, fatal=False): cmd = ['apt-get', '--assume-yes'] cmd.extend(options) cmd.append('install') - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -182,7 +184,7 @@ def apt_update(fatal=False): def apt_purge(packages, fatal=False): """Purge one or more packages""" cmd = ['apt-get', '--assume-yes', 'purge'] - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -193,7 +195,7 @@ def apt_purge(packages, fatal=False): def apt_hold(packages, fatal=False): """Hold one or more packages""" cmd = ['apt-mark', 'hold'] - if isinstance(packages, basestring): + if isinstance(packages, six.string_types): cmd.append(packages) else: cmd.extend(packages) @@ -260,7 +262,7 @@ def add_source(source, key=None): if key: if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: - with NamedTemporaryFile() as key_file: + with NamedTemporaryFile('w+') as key_file: key_file.write(key) key_file.flush() key_file.seek(0) @@ -297,14 +299,14 @@ def configure_sources(update=False, sources = safe_load((config(sources_var) or '').strip()) or [] keys = safe_load((config(keys_var) or '').strip()) or None - if isinstance(sources, basestring): + if isinstance(sources, six.string_types): sources = [sources] if keys is None: for source in sources: add_source(source, None) else: - if isinstance(keys, basestring): + if isinstance(keys, six.string_types): keys = [keys] if len(sources) != len(keys): @@ -401,7 +403,7 @@ def _run_apt_command(cmd, fatal=False): while result is None or result == APT_NO_LOCK: try: result = subprocess.check_call(cmd, env=env) - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: retry_count = retry_count + 1 if retry_count > APT_NO_LOCK_RETRY_COUNT: raise diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 8c04565..8a4624b 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,8 +1,23 @@ import os -import urllib2 -from urllib import urlretrieve -import urlparse import hashlib +import re + +import six +if six.PY3: + from urllib.request import ( + build_opener, install_opener, urlopen, urlretrieve, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ) + from urllib.parse import urlparse, urlunparse, parse_qs + from urllib.error import URLError +else: + from urllib import urlretrieve + from urllib2 import ( + build_opener, install_opener, urlopen, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + URLError + ) + from urlparse import urlparse, urlunparse, parse_qs from charmhelpers.fetch import ( BaseFetchHandler, @@ -15,6 +30,24 @@ from charmhelpers.payload.archive import ( from charmhelpers.core.host import mkdir, check_hash +def splituser(host): + '''urllib.splituser(), but six's support of this seems broken''' + _userprog = re.compile('^(.*)@(.*)$') + match = _userprog.match(host) + if match: + return match.group(1, 2) + return None, host + + +def splitpasswd(user): + '''urllib.splitpasswd(), but six's support of this is missing''' + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) + match = _passwdprog.match(user) + if match: + return match.group(1, 2) + return user, None + + class ArchiveUrlFetchHandler(BaseFetchHandler): """ Handler to download archive files from arbitrary URLs. @@ -42,20 +75,20 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): """ # propogate all exceptions # URLError, OSError, etc - proto, netloc, path, params, query, fragment = urlparse.urlparse(source) + proto, netloc, path, params, query, fragment = urlparse(source) if proto in ('http', 'https'): - auth, barehost = urllib2.splituser(netloc) + auth, barehost = splituser(netloc) if auth is not None: - source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) - username, password = urllib2.splitpasswd(auth) - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + source = urlunparse((proto, barehost, path, params, query, fragment)) + username, password = splitpasswd(auth) + passman = HTTPPasswordMgrWithDefaultRealm() # Realm is set to None in add_password to force the username and password # to be used whatever the realm passman.add_password(None, source, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - response = urllib2.urlopen(source) + authhandler = HTTPBasicAuthHandler(passman) + opener = build_opener(authhandler) + install_opener(opener) + response = urlopen(source) try: with open(dest, 'w') as dest_file: dest_file.write(response.read()) @@ -91,17 +124,21 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): url_parts = self.parse_url(source) dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0755) + mkdir(dest_dir, perms=0o755) dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) try: self.download(source, dld_file) - except urllib2.URLError as e: + except URLError as e: raise UnhandledSource(e.reason) except OSError as e: raise UnhandledSource(e.strerror) - options = urlparse.parse_qs(url_parts.fragment) + options = parse_qs(url_parts.fragment) for key, value in options.items(): - if key in hashlib.algorithms: + if not six.PY3: + algorithms = hashlib.algorithms + else: + algorithms = hashlib.algorithms_available + if key in algorithms: check_hash(dld_file, value, key) if checksum: check_hash(dld_file, checksum, hash_type) diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py index 0e580e4..8ef48f3 100644 --- a/hooks/charmhelpers/fetch/bzrurl.py +++ b/hooks/charmhelpers/fetch/bzrurl.py @@ -5,6 +5,10 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.host import mkdir +import six +if six.PY3: + raise ImportError('bzrlib does not support Python3') + try: from bzrlib.branch import Branch except ImportError: @@ -42,7 +46,7 @@ class BzrUrlFetchHandler(BaseFetchHandler): dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0755) + mkdir(dest_dir, perms=0o755) try: self.branch(source, dest_dir) except OSError as e: diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d67246..f3aa282 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -5,6 +5,10 @@ from charmhelpers.fetch import ( ) from charmhelpers.core.host import mkdir +import six +if six.PY3: + raise ImportError('GitPython does not support Python 3') + try: from git import Repo except ImportError: @@ -17,7 +21,7 @@ class GitUrlFetchHandler(BaseFetchHandler): """Handler for git branches via generic and github URLs""" def can_handle(self, source): url_parts = self.parse_url(source) - #TODO (mattyw) no support for ssh git@ yet + # TODO (mattyw) no support for ssh git@ yet if url_parts.scheme not in ('http', 'https', 'git'): return False else: @@ -30,13 +34,16 @@ class GitUrlFetchHandler(BaseFetchHandler): repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master"): + def install(self, source, branch="master", dest=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", - branch_name) + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0755) + mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch) except OSError as e: diff --git a/hooks/hooks.py b/hooks/hooks.py index 6f15ad4..95a2bf3 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -1143,7 +1143,6 @@ def replica_set_relation_changed(): juju_log('Unable to figure out master... bailing out') return - #Initialize the replicaset #we do this only on the oldest unit in replset #TODO: figure a way how to avoid race conditions - when unit/1 actually diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..883858e --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,5 @@ +coverage>=3.6 +mock>=1.0.1 +nose>=1.3.1 +flake8 + diff --git a/tests/00_setup.sh b/tests/00_setup.sh new file mode 100755 index 0000000..4f58709 --- /dev/null +++ b/tests/00_setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +sudo apt-get install python-setuptools -y +sudo add-apt-repository ppa:juju/stable -y + +sudo apt-get update +sudo apt-get install amulet python3 python3-requests python3-pymongo juju-core charm-tools python-mock python-pymongo -y diff --git a/tests/01_deploy_single.py b/tests/01_deploy_single.py new file mode 100755 index 0000000..718e742 --- /dev/null +++ b/tests/01_deploy_single.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import amulet +from pymongo import MongoClient + +seconds=900 + +d = amulet.Deployment(series='trusty') +d.add('mongodb', charm='mongodb') +d.expose('mongodb') + +# Perform the setup for the deployment. +try: + d.setup(seconds) + d.sentry.wait(seconds) +except amulet.helpers.TimeoutError: + message = 'The environment did not setup in %d seconds.', seconds + amulet.raise_status(amulet.SKIP, msg=message) +except: + raise + +############################################################ +# Validate connectivity from $WORLD +############################################################# +def validate_world_connectivity(): + client = MongoClient(d.sentry.unit['mongodb/0'].info['public-address']) + + db = client['test'] + # Can we successfully insert? + insert_id = db.amulet.insert({'assert': True}) + if insert_id is None: + amulet.raise_status(amulet.FAIL, msg="Failed to insert test data") + # Can we delete from a shard using the Mongos hub? + result = db.amulet.remove(insert_id) + if result['err'] is not None: + amulet.raise_status(amulet.FAIL, msg="Failed to remove test data") + + +validate_world_connectivity() diff --git a/tests/200_deploy.test b/tests/02_deploy_shard_test.py index e196176..b32c687 100755 --- a/tests/200_deploy.test +++ b/tests/02_deploy_shard_test.py @@ -20,7 +20,7 @@ d.add('mongos', charm='mongodb', units=scale) d.add('shard1', charm='mongodb', units=scale) d.add('shard2', charm='mongodb', units=scale) -#Setup the config svr +# Setup the config svr d.configure('configsvr', {'replicaset': 'configsvr'}) # define each shardset @@ -29,10 +29,10 @@ d.configure('shard2', {'replicaset': 'shard2'}) d.configure('mongos', {}) -#Connect the config servers to mongo shell +# Connect the config servers to mongo shell d.relate('configsvr:configsvr', 'mongos:mongos-cfg') -#connect each shard to the mongo shell +# connect each shard to the mongo shell d.relate('mongos:mongos', 'shard1:database') d.relate('mongos:mongos', 'shard2:database') d.expose('configsvr') @@ -74,7 +74,8 @@ def validate_running_services(): output = sentry_dict[service].run('service mongodb status') service_active = str(output).find('mongodb start/running') if service_active == -1: - message = "Failed to find running MongoDB on host {}".format(service) + message = "Failed to find running MongoDB on host {}".format( + service) amulet.raise_status(amulet.SKIP, msg=message) @@ -85,11 +86,11 @@ def validate_world_connectivity(): client = MongoClient(d.sentry.unit['mongos/0'].info['public-address']) db = client['test'] - #Can we successfully insert? + # Can we successfully insert? insert_id = db.amulet.insert({'assert': True}) if insert_id is None: amulet.raise_status(amulet.FAIL, msg="Failed to insert test data") - #Can we delete from a shard using the Mongos hub? + # Can we delete from a shard using the Mongos hub? result = db.amulet.remove(insert_id) if result['err'] is not None: amulet.raise_status(amulet.FAIL, msg="Failed to remove test data") @@ -98,7 +99,7 @@ def validate_world_connectivity(): ############################################################# # Validate relationships ############################################################# -#broken pending 1273312 +# broken pending 1273312 def validate_relationships(): d.sentry.unit['configsvr/0'].relation('configsvr', 'mongos:mongos-cfg') d.sentry.unit['shard1/0'].relation('database', 'mongos:mongos') @@ -122,6 +123,5 @@ def validate_manual_connection(): validate_status_interface() validate_running_services() -#validate_relationships() validate_manual_connection() validate_world_connectivity() diff --git a/tests/10-unit.test b/tests/10-unit.test deleted file mode 100755 index 79e19d3..0000000 --- a/tests/10-unit.test +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/python - -"""Unit test suite.""" - -import os -import sys -import unittest - -runner = unittest.TextTestRunner(verbosity=2) -suite = unittest.TestLoader().discover(os.path.dirname(__file__)) -result = runner.run(suite) -sys.exit(not result.wasSuccessful()) diff --git a/tests/100_configs.test b/tests/100_configs.test deleted file mode 100755 index 58b0132..0000000 --- a/tests/100_configs.test +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh - -set -e - -teardown() { - juju destroy-service mongodb -} -trap teardown EXIT - -juju deploy mongodb -juju expose mongodb - -for try in `seq 1 600` ; do - host=`juju status | tests/get-unit-info mongodb public-address` - - if [ -z "$host" ] ; then - sleep 10 - else - break - fi -done - -if [ -z "$host" ] ; then - echo FAIL: host timed out - exit 1 -fi - -assert_unit_ready() { - for try in `seq 1 600` ; do - status=`juju status | tests/get-unit-info mongodb agent-state` - if [ "$status" != "started" ] ; then - sleep 10 - else - echo "found status as $status" - break - fi - done - - if [ -z "$status" ] ; then - echo FAIL: status timed out - exit 1 - fi -} - -assert_is_listening() { - local port=$1 - tries=$2 - listening="" - for try in `seq 1 $tries` ; do - if ! nc $host $port < /dev/null ; then - continue - fi - listening="27017" - break - done - - if [ -z "$listening" ] ; then - echo "FAIL: not listening on port $port after $tries retries" - return 1 - else - echo "PASS: listening on port $port" - return 0 - fi -} - -assert_unit_ready - -assert_is_listening 27017 10 - -juju set mongodb port=55555 -assert_is_listening 55555 200000 -echo PASS: config changes tests passed. -exit 0 diff --git a/tests/200_relate_ceilometer.test b/tests/50_relate_ceilometer_test.py index aa87b97..b5ff2e5 100755 --- a/tests/200_relate_ceilometer.test +++ b/tests/50_relate_ceilometer_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import amulet -import pdb + class TestDeploy(object): diff --git a/tests/get-unit-info b/tests/get-unit-info deleted file mode 100755 index f5ef103..0000000 --- a/tests/get-unit-info +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/python -# machines: -# 0: {dns-name: ec2-50-17-84-127.compute-1.amazonaws.com, instance-id: i-8c5c3fec} -# 1: {dns-name: ec2-184-73-102-113.compute-1.amazonaws.com, instance-id: i-14a2c174} -# 2: {dns-name: ec2-75-101-184-93.compute-1.amazonaws.com, instance-id: i-e0a2c180} -# services: -# mysql: -# charm: local:mysql-11 -# relations: {db: wordpress} -# units: -# mysql/0: -# machine: 2 -# relations: -# db: {state: up} -# state: started -# wordpress: -# charm: local:wordpress-31 -# exposed: true -# relations: {db: mysql} -# units: -# wordpress/0: -# machine: 1 -# open-ports: [] -# relations: {} -# state: null - -import yaml -import sys -from subprocess import Popen, PIPE - - -def main(): - d = yaml.safe_load(Popen(['juju','status'],stdout=PIPE).stdout) - srv = d.get("services", {}).get(sys.argv[1]) - if srv is None: - return - units = srv.get("units", {}) - if units is None: - return - item = units.items()[0][1].get(sys.argv[2]) - if item is None: - return - print item - -if __name__ == "__main__": - main() diff --git a/tests/test_write_log_rotate_config.py b/unit_tests/test_write_log_rotate_config.py index e1d6154..e1d6154 100644 --- a/tests/test_write_log_rotate_config.py +++ b/unit_tests/test_write_log_rotate_config.py |