Syncing to get closer with lp:~mariosplivalo/charms/trusty/mongodb/charmhelpers-sync

22 files changed, 272 insertions, 232 deletions

diff --git a/Makefile b/Makefile
index 2dee964..27bb822 100644
--- a/Makefile
+++ b/Makefile
@@ -15,15 +15,33 @@
 
 PYTHON := /usr/bin/env python
 
-unittest:
-	tests/10-unit.test
+clean:
+	rm -f .coverage
+	find . -name '*.pyc' -delete
+	rm -rf .venv
+	(which dh_clean && dh_clean) || true
+
+.venv:
+	sudo apt-get install -y gcc python-dev python-virtualenv python-apt
+	virtualenv .venv --system-site-packages
+	.venv/bin/pip install -I -r test_requirements.txt
+
+lint: .venv
+	.venv/bin/flake8 --exclude hooks/charmhelpers hooks tests unit_tests
+
+test: .venv
+	@echo Starting unit tests...
+	.venv/bin/nosetests -s --nologcapture --with-coverage $(EXTRA) unit_tests/
+
+functional_test:
+	@echo Starting amulet tests...
+	@juju test -v -p AMULET_HTTP_PROXY --timeout 900
 
 sync:
	@mkdir -p bin
-	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > bin/charm_helpers_sync.py
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > bin/charm_helpers_sync.py
	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml
 
-clean:
-	@find . -name \*.pyc -delete
-	@find . -name '*.bak' -delete
-
+publish: lint unit_test
+	bzr push lp:charms/mongodb
+	bzr push lp:charms/trusty/mongodb
diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml
index 7e24965..7a71714 100644
--- a/charm-helpers-sync.yaml
+++ b/charm-helpers-sync.yaml
@@ -3,4 +3,4 @@ destination: hooks/charmhelpers
 include:
 - core
 - fetch
- - contrib.hahelpers.cluster \ No newline at end of file
+ - contrib.hahelpers.cluster
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index 6d97200..52ce4b7 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -13,9 +13,10 @@ clustering-related helpers.
 
 import subprocess
 import os
-
 from socket import gethostname as get_unit_hostname
 
+import six
+
 from charmhelpers.core.hookenv import (
 log,
 relation_ids,
@@ -77,7 +78,7 @@ def is_crm_leader(resource):
 "show", resource
 ]
 try:
- status = subprocess.check_output(cmd)
+ status = subprocess.check_output(cmd).decode('UTF-8')
 except subprocess.CalledProcessError:
 return False
 else:
@@ -150,34 +151,42 @@ def https():
 return False
 
 
-def determine_api_port(public_port):
+def determine_api_port(public_port, singlenode_mode=False):
 '''
 Determine correct API server listening port based on
 existence of HTTPS reverse proxy and/or haproxy.
 
 public_port: int: standard public port for given service
 
+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
 returns: int: the correct listening port for the API service
 '''
 i = 0
- if len(peer_units()) > 0 or is_clustered():
+ if singlenode_mode:
+ i += 1
+ elif len(peer_units()) > 0 or is_clustered():
 i += 1
 if https():
 i += 1
 return public_port - (i * 10)
 
 
-def determine_apache_port(public_port):
+def determine_apache_port(public_port, singlenode_mode=False):
 '''
 Description: Determine correct apache listening port based on public IP +
 state of the cluster.
 
 public_port: int: standard public port for given service
 
+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
 returns: int: the correct listening port for the HAProxy service
 '''
 i = 0
- if len(peer_units()) > 0 or is_clustered():
+ if singlenode_mode:
+ i += 1
+ elif len(peer_units()) > 0 or is_clustered():
 i += 1
 return public_port - (i * 10)
 
@@ -197,7 +206,7 @@ def get_hacluster_config():
 for setting in settings:
 conf[setting] = config_get(setting)
 missing = []
- [missing.append(s) for s, v in conf.iteritems() if v is None]
+ [missing.append(s) for s, v in six.iteritems(conf) if v is None]
 if missing:
 log('Insufficient config data to configure hacluster.', level=ERROR)
 raise HAIncompleteConfig
diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py
index cfaf0a6..0adf0db 100644
--- a/hooks/charmhelpers/core/fstab.py
+++ b/hooks/charmhelpers/core/fstab.py
@@ -3,10 +3,11 @@
 
 __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 
+import io
 import os
 
 
-class Fstab(file):
+class Fstab(io.FileIO):
 """This class extends file in order to implement a file reader/writer
 for file `/etc/fstab`
 """
@@ -24,8 +25,8 @@ class Fstab(file):
 options = "defaults"
 
 self.options = options
- self.d = d
- self.p = p
+ self.d = int(d)
+ self.p = int(p)
 
 def __eq__(self, o):
 return str(self) == str(o)
@@ -45,7 +46,7 @@ class Fstab(file):
 self._path = path
 else:
 self._path = self.DEFAULT_PATH
- file.__init__(self, self._path, 'r+')
+ super(Fstab, self).__init__(self._path, 'rb+')
 
 def _hydrate_entry(self, line):
 # NOTE: use split with no arguments to split on any
@@ -58,8 +59,9 @@ class Fstab(file):
 def entries(self):
 self.seek(0)
 for line in self.readlines():
+ line = line.decode('us-ascii')
 try:
- if not line.startswith("#"):
+ if line.strip() and not line.startswith("#"):
 yield self._hydrate_entry(line)
 except ValueError:
 pass
@@ -75,14 +77,14 @@ class Fstab(file):
 if self.get_entry_by_attr('device', entry.device):
 return False
 
- self.write(str(entry) + '\n')
+ self.write((str(entry) + '\n').encode('us-ascii'))
 self.truncate()
 return entry
 
 def remove_entry(self, entry):
 self.seek(0)
 
- lines = self.readlines()
+ lines = [l.decode('us-ascii') for l in self.readlines()]
 
 found = False
 for index, line in enumerate(lines):
@@ -97,7 +99,7 @@ class Fstab(file):
 lines.remove(line)
 
 self.seek(0)
- self.write(''.join(lines))
+ self.write(''.join(lines).encode('us-ascii'))
 self.truncate()
 return True
 
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 083a709..07d1f69 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -9,9 +9,14 @@ import json
 import yaml
 import subprocess
 import sys
-import UserDict
 from subprocess import CalledProcessError
 
+import six
+if not six.PY3:
+ from UserDict import UserDict
+else:
+ from collections import UserDict
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -63,16 +68,18 @@ def log(message, level=None):
 command = ['juju-log']
 if level:
 command += ['-l', level]
+ if not isinstance(message, six.string_types):
+ message = repr(message)
 command += [message]
 subprocess.call(command)
 
 
-class Serializable(UserDict.IterableUserDict):
+class Serializable(UserDict):
 """Wrapper, an object that can be serialized to yaml or json"""
 
 def __init__(self, obj):
 # wrap the object
- UserDict.IterableUserDict.__init__(self)
+ UserDict.__init__(self)
 self.data = obj
 
 def __getattr__(self, attr):
@@ -218,7 +225,7 @@ class Config(dict):
 prev_keys = []
 if self._prev_dict is not None:
 prev_keys = self._prev_dict.keys()
- return list(set(prev_keys + dict.keys(self)))
+ return list(set(prev_keys + list(dict.keys(self))))
 
 def load_previous(self, path=None):
 """Load previous copy of config from disk.
@@ -269,7 +276,7 @@ class Config(dict):
 
 """
 if self._prev_dict:
- for k, v in self._prev_dict.iteritems():
+ for k, v in six.iteritems(self._prev_dict):
 if k not in self:
 self[k] = v
 with open(self.path, 'w') as f:
@@ -284,7 +291,8 @@ def config(scope=None):
 config_cmd_line.append(scope)
 config_cmd_line.append('--format=json')
 try:
- config_data = json.loads(subprocess.check_output(config_cmd_line))
+ config_data = json.loads(
+ subprocess.check_output(config_cmd_line).decode('UTF-8'))
 if scope is not None:
 return config_data
 return Config(config_data)
@@ -303,10 +311,10 @@ def relation_get(attribute=None, unit=None, rid=None):
 if unit:
 _args.append(unit)
 try:
- return json.loads(subprocess.check_output(_args))
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
 except ValueError:
 return None
- except CalledProcessError, e:
+ except CalledProcessError as e:
 if e.returncode == 2:
 return None
 raise
@@ -318,7 +326,7 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
 relation_cmd_line = ['relation-set']
 if relation_id is not None:
 relation_cmd_line.extend(('-r', relation_id))
- for k, v in (relation_settings.items() + kwargs.items()):
+ for k, v in (list(relation_settings.items()) + list(kwargs.items())):
 if v is None:
 relation_cmd_line.append('{}='.format(k))
 else:
@@ -335,7 +343,8 @@ def relation_ids(reltype=None):
 relid_cmd_line = ['relation-ids', '--format=json']
 if reltype is not None:
 relid_cmd_line.append(reltype)
- return json.loads(subprocess.check_output(relid_cmd_line)) or []
+ return json.loads(
+ subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
 return []
 
 
@@ -346,7 +355,8 @@ def related_units(relid=None):
 units_cmd_line = ['relation-list', '--format=json']
 if relid is not None:
 units_cmd_line.extend(('-r', relid))
- return json.loads(subprocess.check_output(units_cmd_line)) or []
+ return json.loads(
+ subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
 
 
 @cached
@@ -455,7 +465,7 @@ def unit_get(attribute):
 """Get the unit ID for the remote unit"""
 _args = ['unit-get', '--format=json', attribute]
 try:
- return json.loads(subprocess.check_output(_args))
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
 except ValueError:
 return None
 
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 0b8bdc5..c6f1680 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -14,11 +14,12 @@ import string
 import subprocess
 import hashlib
 from contextlib import contextmanager
-
 from collections import OrderedDict
 
-from hookenv import log
-from fstab import Fstab
+import six
+
+from .hookenv import log
+from .fstab import Fstab
 
 
 def service_start(service_name):
@@ -54,7 +55,9 @@ def service(action, service_name):
 def service_running(service):
 """Determine whether a system service is running"""
 try:
- output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
+ output = subprocess.check_output(
+ ['service', service, 'status'],
+ stderr=subprocess.STDOUT).decode('UTF-8')
 except subprocess.CalledProcessError:
 return False
 else:
@@ -67,7 +70,9 @@ def service_running(service):
 def service_available(service_name):
 """Determine whether a system service is available"""
 try:
- subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
+ subprocess.check_output(
+ ['service', service_name, 'status'],
+ stderr=subprocess.STDOUT).decode('UTF-8')
 except subprocess.CalledProcessError as e:
 return 'unrecognized service' not in e.output
 else:
@@ -96,6 +101,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
 return user_info
 
 
+def add_group(group_name, system_group=False):
+ """Add a group to the system"""
+ try:
+ group_info = grp.getgrnam(group_name)
+ log('group {0} already exists!'.format(group_name))
+ except KeyError:
+ log('creating group {0}'.format(group_name))
+ cmd = ['addgroup']
+ if system_group:
+ cmd.append('--system')
+ else:
+ cmd.extend([
+ '--group',
+ ])
+ cmd.append(group_name)
+ subprocess.check_call(cmd)
+ group_info = grp.getgrnam(group_name)
+ return group_info
+
+
 def add_user_to_group(username, group):
 """Add a user to a group"""
 cmd = [
@@ -115,7 +140,7 @@ def rsync(from_path, to_path, flags='-r', options=None):
 cmd.append(from_path)
 cmd.append(to_path)
 log(" ".join(cmd))
- return subprocess.check_output(cmd).strip()
+ return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
 def symlink(source, destination):
@@ -130,7 +155,7 @@ def symlink(source, destination):
 subprocess.check_call(cmd)
 
 
-def mkdir(path, owner='root', group='root', perms=0555, force=False):
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
 """Create a directory"""
 log("Making dir {} {}:{} {:o}".format(path, owner, group,
 perms))
@@ -146,7 +171,7 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False):
 os.chown(realpath, uid, gid)
 
 
-def write_file(path, content, owner='root', group='root', perms=0444):
+def write_file(path, content, owner='root', group='root', perms=0o444):
 """Create or overwrite a file with the contents of a string"""
 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
 uid = pwd.getpwnam(owner).pw_uid
@@ -177,7 +202,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
 cmd_args.extend([device, mountpoint])
 try:
 subprocess.check_output(cmd_args)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
 return False
 
@@ -191,7 +216,7 @@ def umount(mountpoint, persist=False):
 cmd_args = ['umount', mountpoint]
 try:
 subprocess.check_output(cmd_args)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
 return False
 
@@ -218,8 +243,8 @@ def file_hash(path, hash_type='md5'):
 """
 if os.path.exists(path):
 h = getattr(hashlib, hash_type)()
- with open(path, 'r') as source:
- h.update(source.read()) # IGNORE:E1101 - it does have update
+ with open(path, 'rb') as source:
+ h.update(source.read())
 return h.hexdigest()
 else:
 return None
@@ -297,7 +322,7 @@ def pwgen(length=None):
 if length is None:
 length = random.choice(range(35, 45))
 alphanumeric_chars = [
- l for l in (string.letters + string.digits)
+ l for l in (string.ascii_letters + string.digits)
 if l not in 'l0QD1vAEIOUaeiou']
 random_chars = [
 random.choice(alphanumeric_chars) for _ in range(length)]
@@ -306,14 +331,14 @@ def pwgen(length=None):
 
 def list_nics(nic_type):
 '''Return a list of nics of given type(s)'''
- if isinstance(nic_type, basestring):
+ if isinstance(nic_type, six.string_types):
 int_types = [nic_type]
 else:
 int_types = nic_type
 interfaces = []
 for int_type in int_types:
 cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
- ip_output = subprocess.check_output(cmd).split('\n')
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
 ip_output = (line for line in ip_output if line)
 for line in ip_output:
 if line.split()[1].startswith(int_type):
@@ -335,7 +360,7 @@ def set_nic_mtu(nic, mtu):
 
 def get_nic_mtu(nic):
 cmd = ['ip', 'addr', 'show', nic]
- ip_output = subprocess.check_output(cmd).split('\n')
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
 mtu = ""
 for line in ip_output:
 words = line.split()
@@ -346,7 +371,7 @@ def get_nic_mtu(nic):
 
 def get_nic_hwaddr(nic):
 cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
- ip_output = subprocess.check_output(cmd)
+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
 hwaddr = ""
 words = ip_output.split()
 if 'link/ether' in words:
@@ -363,8 +388,8 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
 
 '''
 import apt_pkg
- from charmhelpers.fetch import apt_cache
 if not pkgcache:
+ from charmhelpers.fetch import apt_cache
 pkgcache = apt_cache()
 pkg = pkgcache[package]
 return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
index 7067b94..163a793 100644
--- a/hooks/charmhelpers/core/services/helpers.py
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -196,7 +196,7 @@ class StoredContext(dict):
 if not os.path.isabs(file_name):
 file_name = os.path.join(hookenv.charm_dir(), file_name)
 with open(file_name, 'w') as file_stream:
- os.fchmod(file_stream.fileno(), 0600)
+ os.fchmod(file_stream.fileno(), 0o600)
 yaml.dump(config_data, file_stream)
 
 def read_context(self, file_name):
@@ -211,15 +211,19 @@ class StoredContext(dict):
 
 class TemplateCallback(ManagerCallback):
 """
- Callback class that will render a Jinja2 template, for use as a ready action.
+ Callback class that will render a Jinja2 template, for use as a ready
+ action.
+
+ :param str source: The template source file, relative to
+ `$CHARM_DIR/templates`
 
- :param str source: The template source file, relative to `$CHARM_DIR/templates`
 :param str target: The target to write the rendered template to
 :param str owner: The owner of the rendered file
 :param str group: The group of the rendered file
 :param int perms: The permissions of the rendered file
 """
- def __init__(self, source, target, owner='root', group='root', perms=0444):
+ def __init__(self, source, target,
+ owner='root', group='root', perms=0o444):
 self.source = source
 self.target = target
 self.owner = owner
diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py
index 2c63885..83133fa 100644
--- a/hooks/charmhelpers/core/templating.py
+++ b/hooks/charmhelpers/core/templating.py
@@ -4,7 +4,8 @@ from charmhelpers.core import host
 from charmhelpers.core import hookenv
 
 
-def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
+def render(source, target, context, owner='root', group='root',
+ perms=0o444, templates_dir=None):
 """
 Render a template.
 
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 2398e8e..0a126fc 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -5,10 +5,6 @@ from yaml import safe_load
 from charmhelpers.core.host import (
 lsb_release
 )
-from urlparse import (
- urlparse,
- urlunparse,
-)
 import subprocess
 from charmhelpers.core.hookenv import (
 config,
@@ -16,6 +12,12 @@ from charmhelpers.core.hookenv import (
 )
 import os
 
+import six
+if six.PY3:
+ from urllib.parse import urlparse, urlunparse
+else:
+ from urlparse import urlparse, urlunparse
+
 
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@@ -149,7 +151,7 @@ def apt_install(packages, options=None, fatal=False):
 cmd = ['apt-get', '--assume-yes']
 cmd.extend(options)
 cmd.append('install')
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -182,7 +184,7 @@ def apt_update(fatal=False):
 def apt_purge(packages, fatal=False):
 """Purge one or more packages"""
 cmd = ['apt-get', '--assume-yes', 'purge']
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -193,7 +195,7 @@ def apt_purge(packages, fatal=False):
 def apt_hold(packages, fatal=False):
 """Hold one or more packages"""
 cmd = ['apt-mark', 'hold']
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -260,7 +262,7 @@ def add_source(source, key=None):
 
 if key:
 if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
- with NamedTemporaryFile() as key_file:
+ with NamedTemporaryFile('w+') as key_file:
 key_file.write(key)
 key_file.flush()
 key_file.seek(0)
@@ -297,14 +299,14 @@ def configure_sources(update=False,
 sources = safe_load((config(sources_var) or '').strip()) or []
 keys = safe_load((config(keys_var) or '').strip()) or None
 
- if isinstance(sources, basestring):
+ if isinstance(sources, six.string_types):
 sources = [sources]
 
 if keys is None:
 for source in sources:
 add_source(source, None)
 else:
- if isinstance(keys, basestring):
+ if isinstance(keys, six.string_types):
 keys = [keys]
 
 if len(sources) != len(keys):
@@ -401,7 +403,7 @@ def _run_apt_command(cmd, fatal=False):
 while result is None or result == APT_NO_LOCK:
 try:
 result = subprocess.check_call(cmd, env=env)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 retry_count = retry_count + 1
 if retry_count > APT_NO_LOCK_RETRY_COUNT:
 raise
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index 8c04565..8a4624b 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -1,8 +1,23 @@
 import os
-import urllib2
-from urllib import urlretrieve
-import urlparse
 import hashlib
+import re
+
+import six
+if six.PY3:
+ from urllib.request import (
+ build_opener, install_opener, urlopen, urlretrieve,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ )
+ from urllib.parse import urlparse, urlunparse, parse_qs
+ from urllib.error import URLError
+else:
+ from urllib import urlretrieve
+ from urllib2 import (
+ build_opener, install_opener, urlopen,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ URLError
+ )
+ from urlparse import urlparse, urlunparse, parse_qs
 
 from charmhelpers.fetch import (
 BaseFetchHandler,
@@ -15,6 +30,24 @@ from charmhelpers.payload.archive import (
 from charmhelpers.core.host import mkdir, check_hash
 
 
+def splituser(host):
+ '''urllib.splituser(), but six's support of this seems broken'''
+ _userprog = re.compile('^(.*)@(.*)$')
+ match = _userprog.match(host)
+ if match:
+ return match.group(1, 2)
+ return None, host
+
+
+def splitpasswd(user):
+ '''urllib.splitpasswd(), but six's support of this is missing'''
+ _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+ match = _passwdprog.match(user)
+ if match:
+ return match.group(1, 2)
+ return user, None
+
+
 class ArchiveUrlFetchHandler(BaseFetchHandler):
 """
 Handler to download archive files from arbitrary URLs.
@@ -42,20 +75,20 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
 """
 # propogate all exceptions
 # URLError, OSError, etc
- proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+ proto, netloc, path, params, query, fragment = urlparse(source)
 if proto in ('http', 'https'):
- auth, barehost = urllib2.splituser(netloc)
+ auth, barehost = splituser(netloc)
 if auth is not None:
- source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
- username, password = urllib2.splitpasswd(auth)
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ source = urlunparse((proto, barehost, path, params, query, fragment))
+ username, password = splitpasswd(auth)
+ passman = HTTPPasswordMgrWithDefaultRealm()
 # Realm is set to None in add_password to force the username and password
 # to be used whatever the realm
 passman.add_password(None, source, username, password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(authhandler)
- urllib2.install_opener(opener)
- response = urllib2.urlopen(source)
+ authhandler = HTTPBasicAuthHandler(passman)
+ opener = build_opener(authhandler)
+ install_opener(opener)
+ response = urlopen(source)
 try:
 with open(dest, 'w') as dest_file:
 dest_file.write(response.read())
@@ -91,17 +124,21 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
 url_parts = self.parse_url(source)
 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
 if not os.path.exists(dest_dir):
- mkdir(dest_dir, perms=0755)
+ mkdir(dest_dir, perms=0o755)
 dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
 try:
 self.download(source, dld_file)
- except urllib2.URLError as e:
+ except URLError as e:
 raise UnhandledSource(e.reason)
 except OSError as e:
 raise UnhandledSource(e.strerror)
- options = urlparse.parse_qs(url_parts.fragment)
+ options = parse_qs(url_parts.fragment)
 for key, value in options.items():
- if key in hashlib.algorithms:
+ if not six.PY3:
+ algorithms = hashlib.algorithms
+ else:
+ algorithms = hashlib.algorithms_available
+ if key in algorithms:
 check_hash(dld_file, value, key)
 if checksum:
 check_hash(dld_file, checksum, hash_type)
diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py
index 0e580e4..8ef48f3 100644
--- a/hooks/charmhelpers/fetch/bzrurl.py
+++ b/hooks/charmhelpers/fetch/bzrurl.py
@@ -5,6 +5,10 @@ from charmhelpers.fetch import (
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+ raise ImportError('bzrlib does not support Python3')
+
 try:
 from bzrlib.branch import Branch
 except ImportError:
@@ -42,7 +46,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
 branch_name)
 if not os.path.exists(dest_dir):
- mkdir(dest_dir, perms=0755)
+ mkdir(dest_dir, perms=0o755)
 try:
 self.branch(source, dest_dir)
 except OSError as e:
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 7d67246..f3aa282 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -5,6 +5,10 @@ from charmhelpers.fetch import (
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+ raise ImportError('GitPython does not support Python 3')
+
 try:
 from git import Repo
 except ImportError:
@@ -17,7 +21,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
 """Handler for git branches via generic and github URLs"""
 def can_handle(self, source):
 url_parts = self.parse_url(source)
- #TODO (mattyw) no support for ssh git@ yet
+ # TODO (mattyw) no support for ssh git@ yet
 if url_parts.scheme not in ('http', 'https', 'git'):
 return False
 else:
@@ -30,13 +34,16 @@ class GitUrlFetchHandler(BaseFetchHandler):
 repo = Repo.clone_from(source, dest)
 repo.git.checkout(branch)
 
- def install(self, source, branch="master"):
+ def install(self, source, branch="master", dest=None):
 url_parts = self.parse_url(source)
 branch_name = url_parts.path.strip("/").split("/")[-1]
- dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
- branch_name)
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
 if not os.path.exists(dest_dir):
- mkdir(dest_dir, perms=0755)
+ mkdir(dest_dir, perms=0o755)
 try:
 self.clone(source, dest_dir, branch)
 except OSError as e:
diff --git a/hooks/hooks.py b/hooks/hooks.py
index 6f15ad4..95a2bf3 100755
--- a/hooks/hooks.py
+++ b/hooks/hooks.py
@@ -1143,7 +1143,6 @@ def replica_set_relation_changed():
 juju_log('Unable to figure out master... bailing out')
 return
 
- 
 #Initialize the replicaset
 #we do this only on the oldest unit in replset
 #TODO: figure a way how to avoid race conditions - when unit/1 actually 
diff --git a/test_requirements.txt b/test_requirements.txt
new file mode 100644
index 0000000..883858e
--- /dev/null
+++ b/test_requirements.txt
@@ -0,0 +1,5 @@
+coverage>=3.6
+mock>=1.0.1
+nose>=1.3.1
+flake8
+
diff --git a/tests/00_setup.sh b/tests/00_setup.sh
new file mode 100755
index 0000000..4f58709
--- /dev/null
+++ b/tests/00_setup.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+sudo apt-get install python-setuptools -y
+sudo add-apt-repository ppa:juju/stable -y
+
+sudo apt-get update
+sudo apt-get install amulet python3 python3-requests python3-pymongo juju-core charm-tools python-mock python-pymongo -y
diff --git a/tests/01_deploy_single.py b/tests/01_deploy_single.py
new file mode 100755
index 0000000..718e742
--- /dev/null
+++ b/tests/01_deploy_single.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import amulet
+from pymongo import MongoClient
+
+seconds=900
+
+d = amulet.Deployment(series='trusty')
+d.add('mongodb', charm='mongodb')
+d.expose('mongodb')
+
+# Perform the setup for the deployment.
+try:
+ d.setup(seconds)
+ d.sentry.wait(seconds)
+except amulet.helpers.TimeoutError:
+ message = 'The environment did not setup in %d seconds.', seconds
+ amulet.raise_status(amulet.SKIP, msg=message)
+except:
+ raise
+
+############################################################
+# Validate connectivity from $WORLD
+#############################################################
+def validate_world_connectivity():
+ client = MongoClient(d.sentry.unit['mongodb/0'].info['public-address'])
+
+ db = client['test']
+ # Can we successfully insert?
+ insert_id = db.amulet.insert({'assert': True})
+ if insert_id is None:
+ amulet.raise_status(amulet.FAIL, msg="Failed to insert test data")
+ # Can we delete from a shard using the Mongos hub?
+ result = db.amulet.remove(insert_id)
+ if result['err'] is not None:
+ amulet.raise_status(amulet.FAIL, msg="Failed to remove test data")
+
+
+validate_world_connectivity()
diff --git a/tests/200_deploy.test b/tests/02_deploy_shard_test.py
index e196176..b32c687 100755
--- a/tests/200_deploy.test
+++ b/tests/02_deploy_shard_test.py
@@ -20,7 +20,7 @@ d.add('mongos', charm='mongodb', units=scale)
 d.add('shard1', charm='mongodb', units=scale)
 d.add('shard2', charm='mongodb', units=scale)
 
-#Setup the config svr
+# Setup the config svr
 d.configure('configsvr', {'replicaset': 'configsvr'})
 
 # define each shardset
@@ -29,10 +29,10 @@ d.configure('shard2', {'replicaset': 'shard2'})
 
 d.configure('mongos', {})
 
-#Connect the config servers to mongo shell
+# Connect the config servers to mongo shell
 d.relate('configsvr:configsvr', 'mongos:mongos-cfg')
 
-#connect each shard to the mongo shell
+# connect each shard to the mongo shell
 d.relate('mongos:mongos', 'shard1:database')
 d.relate('mongos:mongos', 'shard2:database')
 d.expose('configsvr')
@@ -74,7 +74,8 @@ def validate_running_services():
 output = sentry_dict[service].run('service mongodb status')
 service_active = str(output).find('mongodb start/running')
 if service_active == -1:
- message = "Failed to find running MongoDB on host {}".format(service)
+ message = "Failed to find running MongoDB on host {}".format(
+ service)
 amulet.raise_status(amulet.SKIP, msg=message)
 
 
@@ -85,11 +86,11 @@ def validate_world_connectivity():
 client = MongoClient(d.sentry.unit['mongos/0'].info['public-address'])
 
 db = client['test']
- #Can we successfully insert?
+ # Can we successfully insert?
 insert_id = db.amulet.insert({'assert': True})
 if insert_id is None:
 amulet.raise_status(amulet.FAIL, msg="Failed to insert test data")
- #Can we delete from a shard using the Mongos hub?
+ # Can we delete from a shard using the Mongos hub?
 result = db.amulet.remove(insert_id)
 if result['err'] is not None:
 amulet.raise_status(amulet.FAIL, msg="Failed to remove test data")
@@ -98,7 +99,7 @@ def validate_world_connectivity():
 #############################################################
 # Validate relationships
 #############################################################
-#broken pending 1273312
+# broken pending 1273312
 def validate_relationships():
 d.sentry.unit['configsvr/0'].relation('configsvr', 'mongos:mongos-cfg')
 d.sentry.unit['shard1/0'].relation('database', 'mongos:mongos')
@@ -122,6 +123,5 @@ def validate_manual_connection():
 
 validate_status_interface()
 validate_running_services()
-#validate_relationships()
 validate_manual_connection()
 validate_world_connectivity()
diff --git a/tests/10-unit.test b/tests/10-unit.test
deleted file mode 100755
index 79e19d3..0000000
--- a/tests/10-unit.test
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/python
-
-"""Unit test suite."""
-
-import os
-import sys
-import unittest
-
-runner = unittest.TextTestRunner(verbosity=2)
-suite = unittest.TestLoader().discover(os.path.dirname(__file__))
-result = runner.run(suite)
-sys.exit(not result.wasSuccessful())
diff --git a/tests/100_configs.test b/tests/100_configs.test
deleted file mode 100755
index 58b0132..0000000
--- a/tests/100_configs.test
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/bin/sh
-
-set -e
-
-teardown() {
- juju destroy-service mongodb
-}
-trap teardown EXIT
-
-juju deploy mongodb
-juju expose mongodb
-
-for try in `seq 1 600` ; do
- host=`juju status | tests/get-unit-info mongodb public-address`
- 
- if [ -z "$host" ] ; then
- sleep 10
- else
- break
- fi
-done
-
-if [ -z "$host" ] ; then
- echo FAIL: host timed out
- exit 1
-fi
-
-assert_unit_ready() {
- for try in `seq 1 600` ; do
- status=`juju status | tests/get-unit-info mongodb agent-state`
- if [ "$status" != "started" ] ; then
- sleep 10
- else
- echo "found status as $status"
- break
- fi
- done
-
- if [ -z "$status" ] ; then
- echo FAIL: status timed out
- exit 1
- fi
-}
-
-assert_is_listening() {
- local port=$1
- tries=$2 
- listening=""
- for try in `seq 1 $tries` ; do
- if ! nc $host $port < /dev/null ; then
- continue
- fi
- listening="27017"
- break
- done
-
- if [ -z "$listening" ] ; then
- echo "FAIL: not listening on port $port after $tries retries"
- return 1
- else
- echo "PASS: listening on port $port"
- return 0
- fi
-}
-
-assert_unit_ready
-
-assert_is_listening 27017 10
-
-juju set mongodb port=55555
-assert_is_listening 55555 200000
-echo PASS: config changes tests passed.
-exit 0
diff --git a/tests/200_relate_ceilometer.test b/tests/50_relate_ceilometer_test.py
index aa87b97..b5ff2e5 100755
--- a/tests/200_relate_ceilometer.test
+++ b/tests/50_relate_ceilometer_test.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 import amulet
-import pdb
+
 
 class TestDeploy(object):
 
diff --git a/tests/get-unit-info b/tests/get-unit-info
deleted file mode 100755
index f5ef103..0000000
--- a/tests/get-unit-info
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/python
-# machines:
-# 0: {dns-name: ec2-50-17-84-127.compute-1.amazonaws.com, instance-id: i-8c5c3fec}
-# 1: {dns-name: ec2-184-73-102-113.compute-1.amazonaws.com, instance-id: i-14a2c174}
-# 2: {dns-name: ec2-75-101-184-93.compute-1.amazonaws.com, instance-id: i-e0a2c180}
-# services:
-# mysql:
-# charm: local:mysql-11
-# relations: {db: wordpress}
-# units:
-# mysql/0:
-# machine: 2
-# relations:
-# db: {state: up}
-# state: started
-# wordpress:
-# charm: local:wordpress-31
-# exposed: true
-# relations: {db: mysql}
-# units:
-# wordpress/0:
-# machine: 1
-# open-ports: []
-# relations: {}
-# state: null
-
-import yaml
-import sys
-from subprocess import Popen, PIPE
-
-
-def main():
- d = yaml.safe_load(Popen(['juju','status'],stdout=PIPE).stdout)
- srv = d.get("services", {}).get(sys.argv[1])
- if srv is None:
- return
- units = srv.get("units", {})
- if units is None:
- return
- item = units.items()[0][1].get(sys.argv[2])
- if item is None:
- return
- print item
-
-if __name__ == "__main__":
- main()
diff --git a/tests/test_write_log_rotate_config.py b/unit_tests/test_write_log_rotate_config.py
index e1d6154..e1d6154 100644
--- a/tests/test_write_log_rotate_config.py
+++ b/unit_tests/test_write_log_rotate_config.py