Added contrib.hahelpers.cluster, synced charmhelpers.

15 files changed, 1309 insertions, 101 deletions

diff --git a/hooks/charmhelpers/contrib/__init__.py b/hooks/charmhelpers/contrib/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/hooks/charmhelpers/contrib/__init__.py
diff --git a/hooks/charmhelpers/contrib/hahelpers/__init__.py b/hooks/charmhelpers/contrib/hahelpers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/hooks/charmhelpers/contrib/hahelpers/__init__.py
diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
new file mode 100644
index 0000000..52ce4b7
--- /dev/null
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -0,0 +1,234 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+# James Page <james.page@ubuntu.com>
+# Adam Gandelman <adamg@ubuntu.com>
+#
+
+"""
+Helpers for clustering and determining "cluster leadership" and other
+clustering-related helpers.
+"""
+
+import subprocess
+import os
+from socket import gethostname as get_unit_hostname
+
+import six
+
+from charmhelpers.core.hookenv import (
+ log,
+ relation_ids,
+ related_units as relation_list,
+ relation_get,
+ config as config_get,
+ INFO,
+ ERROR,
+ WARNING,
+ unit_get,
+)
+
+
+class HAIncompleteConfig(Exception):
+ pass
+
+
+def is_elected_leader(resource):
+ """
+ Returns True if the charm executing this is the elected cluster leader.
+
+ It relies on two mechanisms to determine leadership:
+ 1. If the charm is part of a corosync cluster, call corosync to
+ determine leadership.
+ 2. If the charm is not part of a corosync cluster, the leader is
+ determined as being "the alive unit with the lowest unit numer". In
+ other words, the oldest surviving unit.
+ """
+ if is_clustered():
+ if not is_crm_leader(resource):
+ log('Deferring action to CRM leader.', level=INFO)
+ return False
+ else:
+ peers = peer_units()
+ if peers and not oldest_peer(peers):
+ log('Deferring action to oldest service unit.', level=INFO)
+ return False
+ return True
+
+
+def is_clustered():
+ for r_id in (relation_ids('ha') or []):
+ for unit in (relation_list(r_id) or []):
+ clustered = relation_get('clustered',
+ rid=r_id,
+ unit=unit)
+ if clustered:
+ return True
+ return False
+
+
+def is_crm_leader(resource):
+ """
+ Returns True if the charm calling this is the elected corosync leader,
+ as returned by calling the external "crm" command.
+ """
+ cmd = [
+ "crm", "resource",
+ "show", resource
+ ]
+ try:
+ status = subprocess.check_output(cmd).decode('UTF-8')
+ except subprocess.CalledProcessError:
+ return False
+ else:
+ if get_unit_hostname() in status:
+ return True
+ else:
+ return False
+
+
+def is_leader(resource):
+ log("is_leader is deprecated. Please consider using is_crm_leader "
+ "instead.", level=WARNING)
+ return is_crm_leader(resource)
+
+
+def peer_units(peer_relation="cluster"):
+ peers = []
+ for r_id in (relation_ids(peer_relation) or []):
+ for unit in (relation_list(r_id) or []):
+ peers.append(unit)
+ return peers
+
+
+def peer_ips(peer_relation='cluster', addr_key='private-address'):
+ '''Return a dict of peers and their private-address'''
+ peers = {}
+ for r_id in relation_ids(peer_relation):
+ for unit in relation_list(r_id):
+ peers[unit] = relation_get(addr_key, rid=r_id, unit=unit)
+ return peers
+
+
+def oldest_peer(peers):
+ """Determines who the oldest peer is by comparing unit numbers."""
+ local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
+ for peer in peers:
+ remote_unit_no = int(peer.split('/')[1])
+ if remote_unit_no < local_unit_no:
+ return False
+ return True
+
+
+def eligible_leader(resource):
+ log("eligible_leader is deprecated. Please consider using "
+ "is_elected_leader instead.", level=WARNING)
+ return is_elected_leader(resource)
+
+
+def https():
+ '''
+ Determines whether enough data has been provided in configuration
+ or relation data to configure HTTPS
+ .
+ returns: boolean
+ '''
+ if config_get('use-https') == "yes":
+ return True
+ if config_get('ssl_cert') and config_get('ssl_key'):
+ return True
+ for r_id in relation_ids('identity-service'):
+ for unit in relation_list(r_id):
+ # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN
+ rel_state = [
+ relation_get('https_keystone', rid=r_id, unit=unit),
+ relation_get('ca_cert', rid=r_id, unit=unit),
+ ]
+ # NOTE: works around (LP: #1203241)
+ if (None not in rel_state) and ('' not in rel_state):
+ return True
+ return False
+
+
+def determine_api_port(public_port, singlenode_mode=False):
+ '''
+ Determine correct API server listening port based on
+ existence of HTTPS reverse proxy and/or haproxy.
+
+ public_port: int: standard public port for given service
+
+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
+ returns: int: the correct listening port for the API service
+ '''
+ i = 0
+ if singlenode_mode:
+ i += 1
+ elif len(peer_units()) > 0 or is_clustered():
+ i += 1
+ if https():
+ i += 1
+ return public_port - (i * 10)
+
+
+def determine_apache_port(public_port, singlenode_mode=False):
+ '''
+ Description: Determine correct apache listening port based on public IP +
+ state of the cluster.
+
+ public_port: int: standard public port for given service
+
+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
+ returns: int: the correct listening port for the HAProxy service
+ '''
+ i = 0
+ if singlenode_mode:
+ i += 1
+ elif len(peer_units()) > 0 or is_clustered():
+ i += 1
+ return public_port - (i * 10)
+
+
+def get_hacluster_config():
+ '''
+ Obtains all relevant configuration from charm configuration required
+ for initiating a relation to hacluster:
+
+ ha-bindiface, ha-mcastport, vip
+
+ returns: dict: A dict containing settings keyed by setting name.
+ raises: HAIncompleteConfig if settings are missing.
+ '''
+ settings = ['ha-bindiface', 'ha-mcastport', 'vip']
+ conf = {}
+ for setting in settings:
+ conf[setting] = config_get(setting)
+ missing = []
+ [missing.append(s) for s, v in six.iteritems(conf) if v is None]
+ if missing:
+ log('Insufficient config data to configure hacluster.', level=ERROR)
+ raise HAIncompleteConfig
+ return conf
+
+
+def canonical_url(configs, vip_setting='vip'):
+ '''
+ Returns the correct HTTP URL to this host given the state of HTTPS
+ configuration and hacluster.
+
+ :configs : OSTemplateRenderer: A config tempating object to inspect for
+ a complete https context.
+
+ :vip_setting: str: Setting in charm config that specifies
+ VIP address.
+ '''
+ scheme = 'http'
+ if 'https' in configs.complete_contexts():
+ scheme = 'https'
+ if is_clustered():
+ addr = config_get(vip_setting)
+ else:
+ addr = unit_get('private-address')
+ return '%s://%s' % (scheme, addr)
diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py
index cfaf0a6..0adf0db 100644
--- a/hooks/charmhelpers/core/fstab.py
+++ b/hooks/charmhelpers/core/fstab.py
@@ -3,10 +3,11 @@
 
 __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
 
+import io
 import os
 
 
-class Fstab(file):
+class Fstab(io.FileIO):
 """This class extends file in order to implement a file reader/writer
 for file `/etc/fstab`
 """
@@ -24,8 +25,8 @@ class Fstab(file):
 options = "defaults"
 
 self.options = options
- self.d = d
- self.p = p
+ self.d = int(d)
+ self.p = int(p)
 
 def __eq__(self, o):
 return str(self) == str(o)
@@ -45,7 +46,7 @@ class Fstab(file):
 self._path = path
 else:
 self._path = self.DEFAULT_PATH
- file.__init__(self, self._path, 'r+')
+ super(Fstab, self).__init__(self._path, 'rb+')
 
 def _hydrate_entry(self, line):
 # NOTE: use split with no arguments to split on any
@@ -58,8 +59,9 @@ class Fstab(file):
 def entries(self):
 self.seek(0)
 for line in self.readlines():
+ line = line.decode('us-ascii')
 try:
- if not line.startswith("#"):
+ if line.strip() and not line.startswith("#"):
 yield self._hydrate_entry(line)
 except ValueError:
 pass
@@ -75,14 +77,14 @@ class Fstab(file):
 if self.get_entry_by_attr('device', entry.device):
 return False
 
- self.write(str(entry) + '\n')
+ self.write((str(entry) + '\n').encode('us-ascii'))
 self.truncate()
 return entry
 
 def remove_entry(self, entry):
 self.seek(0)
 
- lines = self.readlines()
+ lines = [l.decode('us-ascii') for l in self.readlines()]
 
 found = False
 for index, line in enumerate(lines):
@@ -97,7 +99,7 @@ class Fstab(file):
 lines.remove(line)
 
 self.seek(0)
- self.write(''.join(lines))
+ self.write(''.join(lines).encode('us-ascii'))
 self.truncate()
 return True
 
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index c953043..07d1f69 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -9,9 +9,14 @@ import json
 import yaml
 import subprocess
 import sys
-import UserDict
 from subprocess import CalledProcessError
 
+import six
+if not six.PY3:
+ from UserDict import UserDict
+else:
+ from collections import UserDict
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -63,16 +68,18 @@ def log(message, level=None):
 command = ['juju-log']
 if level:
 command += ['-l', level]
+ if not isinstance(message, six.string_types):
+ message = repr(message)
 command += [message]
 subprocess.call(command)
 
 
-class Serializable(UserDict.IterableUserDict):
+class Serializable(UserDict):
 """Wrapper, an object that can be serialized to yaml or json"""
 
 def __init__(self, obj):
 # wrap the object
- UserDict.IterableUserDict.__init__(self)
+ UserDict.__init__(self)
 self.data = obj
 
 def __getattr__(self, attr):
@@ -156,12 +163,15 @@ def hook_name():
 
 
 class Config(dict):
- """A Juju charm config dictionary that can write itself to
- disk (as json) and track which values have changed since
- the previous hook invocation.
+ """A dictionary representation of the charm's config.yaml, with some
+ extra features:
+
+ - See which values in the dictionary have changed since the previous hook.
+ - For values that have changed, see what the previous value was.
+ - Store arbitrary data for use in a later hook.
 
- Do not instantiate this object directly - instead call
- ``hookenv.config()``
+ NOTE: Do not instantiate this object directly - instead call
+ ``hookenv.config()``, which will return an instance of :class:`Config`.
 
 Example usage::
 
@@ -170,8 +180,8 @@ class Config(dict):
 >>> config = hookenv.config()
 >>> config['foo']
 'bar'
+ >>> # store a new key/value for later use
 >>> config['mykey'] = 'myval'
- >>> config.save()
 
 
 >>> # user runs `juju set mycharm foo=baz`
@@ -188,22 +198,40 @@ class Config(dict):
 >>> # keys/values that we add are preserved across hooks
 >>> config['mykey']
 'myval'
- >>> # don't forget to save at the end of hook!
- >>> config.save()
 
 """
 CONFIG_FILE_NAME = '.juju-persistent-config'
 
 def __init__(self, *args, **kw):
 super(Config, self).__init__(*args, **kw)
+ self.implicit_save = True
 self._prev_dict = None
 self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
 if os.path.exists(self.path):
 self.load_previous()
 
+ def __getitem__(self, key):
+ """For regular dict lookups, check the current juju config first,
+ then the previous (saved) copy. This ensures that user-saved values
+ will be returned by a dict lookup.
+
+ """
+ try:
+ return dict.__getitem__(self, key)
+ except KeyError:
+ return (self._prev_dict or {})[key]
+
+ def keys(self):
+ prev_keys = []
+ if self._prev_dict is not None:
+ prev_keys = self._prev_dict.keys()
+ return list(set(prev_keys + list(dict.keys(self))))
+
 def load_previous(self, path=None):
- """Load previous copy of config from disk so that current values
- can be compared to previous values.
+ """Load previous copy of config from disk.
+
+ In normal usage you don't need to call this method directly - it
+ is called automatically at object initialization.
 
 :param path:
 
@@ -218,8 +246,8 @@ class Config(dict):
 self._prev_dict = json.load(f)
 
 def changed(self, key):
- """Return true if the value for this key has changed since
- the last save.
+ """Return True if the current value for this key is different from
+ the previous value.
 
 """
 if self._prev_dict is None:
@@ -228,7 +256,7 @@ class Config(dict):
 
 def previous(self, key):
 """Return previous value for this key, or None if there
- is no "previous" value.
+ is no previous value.
 
 """
 if self._prev_dict:
@@ -238,11 +266,17 @@ class Config(dict):
 def save(self):
 """Save this config to disk.
 
- Preserves items in _prev_dict that do not exist in self.
+ If the charm is using the :mod:`Services Framework <services.base>`
+ or :meth:'@hook <Hooks.hook>' decorator, this
+ is called automatically at the end of successful hook execution.
+ Otherwise, it should be called directly by user code.
+
+ To disable automatic saves, set ``implicit_save=False`` on this
+ instance.
 
 """
 if self._prev_dict:
- for k, v in self._prev_dict.iteritems():
+ for k, v in six.iteritems(self._prev_dict):
 if k not in self:
 self[k] = v
 with open(self.path, 'w') as f:
@@ -257,7 +291,8 @@ def config(scope=None):
 config_cmd_line.append(scope)
 config_cmd_line.append('--format=json')
 try:
- config_data = json.loads(subprocess.check_output(config_cmd_line))
+ config_data = json.loads(
+ subprocess.check_output(config_cmd_line).decode('UTF-8'))
 if scope is not None:
 return config_data
 return Config(config_data)
@@ -276,21 +311,22 @@ def relation_get(attribute=None, unit=None, rid=None):
 if unit:
 _args.append(unit)
 try:
- return json.loads(subprocess.check_output(_args))
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
 except ValueError:
 return None
- except CalledProcessError, e:
+ except CalledProcessError as e:
 if e.returncode == 2:
 return None
 raise
 
 
-def relation_set(relation_id=None, relation_settings={}, **kwargs):
+def relation_set(relation_id=None, relation_settings=None, **kwargs):
 """Set relation information for the current unit"""
+ relation_settings = relation_settings if relation_settings else {}
 relation_cmd_line = ['relation-set']
 if relation_id is not None:
 relation_cmd_line.extend(('-r', relation_id))
- for k, v in (relation_settings.items() + kwargs.items()):
+ for k, v in (list(relation_settings.items()) + list(kwargs.items())):
 if v is None:
 relation_cmd_line.append('{}='.format(k))
 else:
@@ -307,7 +343,8 @@ def relation_ids(reltype=None):
 relid_cmd_line = ['relation-ids', '--format=json']
 if reltype is not None:
 relid_cmd_line.append(reltype)
- return json.loads(subprocess.check_output(relid_cmd_line)) or []
+ return json.loads(
+ subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
 return []
 
 
@@ -318,7 +355,8 @@ def related_units(relid=None):
 units_cmd_line = ['relation-list', '--format=json']
 if relid is not None:
 units_cmd_line.extend(('-r', relid))
- return json.loads(subprocess.check_output(units_cmd_line)) or []
+ return json.loads(
+ subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
 
 
 @cached
@@ -427,7 +465,7 @@ def unit_get(attribute):
 """Get the unit ID for the remote unit"""
 _args = ['unit-get', '--format=json', attribute]
 try:
- return json.loads(subprocess.check_output(_args))
+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
 except ValueError:
 return None
 
@@ -464,9 +502,10 @@ class Hooks(object):
 hooks.execute(sys.argv)
 """
 
- def __init__(self):
+ def __init__(self, config_save=True):
 super(Hooks, self).__init__()
 self._hooks = {}
+ self._config_save = config_save
 
 def register(self, name, function):
 """Register a hook"""
@@ -477,6 +516,10 @@ class Hooks(object):
 hook_name = os.path.basename(args[0])
 if hook_name in self._hooks:
 self._hooks[hook_name]()
+ if self._config_save:
+ cfg = config()
+ if cfg.implicit_save:
+ cfg.save()
 else:
 raise UnregisteredHookError(hook_name)
 
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 8b617a4..c6f1680 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -6,17 +6,20 @@
 # Matthew Wedgwood <matthew.wedgwood@canonical.com>
 
 import os
+import re
 import pwd
 import grp
 import random
 import string
 import subprocess
 import hashlib
-
+from contextlib import contextmanager
 from collections import OrderedDict
 
-from hookenv import log
-from fstab import Fstab
+import six
+
+from .hookenv import log
+from .fstab import Fstab
 
 
 def service_start(service_name):
@@ -52,7 +55,9 @@ def service(action, service_name):
 def service_running(service):
 """Determine whether a system service is running"""
 try:
- output = subprocess.check_output(['service', service, 'status'])
+ output = subprocess.check_output(
+ ['service', service, 'status'],
+ stderr=subprocess.STDOUT).decode('UTF-8')
 except subprocess.CalledProcessError:
 return False
 else:
@@ -62,6 +67,18 @@ def service_running(service):
 return False
 
 
+def service_available(service_name):
+ """Determine whether a system service is available"""
+ try:
+ subprocess.check_output(
+ ['service', service_name, 'status'],
+ stderr=subprocess.STDOUT).decode('UTF-8')
+ except subprocess.CalledProcessError as e:
+ return 'unrecognized service' not in e.output
+ else:
+ return True
+
+
 def adduser(username, password=None, shell='/bin/bash', system_user=False):
 """Add a user to the system"""
 try:
@@ -84,6 +101,26 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False):
 return user_info
 
 
+def add_group(group_name, system_group=False):
+ """Add a group to the system"""
+ try:
+ group_info = grp.getgrnam(group_name)
+ log('group {0} already exists!'.format(group_name))
+ except KeyError:
+ log('creating group {0}'.format(group_name))
+ cmd = ['addgroup']
+ if system_group:
+ cmd.append('--system')
+ else:
+ cmd.extend([
+ '--group',
+ ])
+ cmd.append(group_name)
+ subprocess.check_call(cmd)
+ group_info = grp.getgrnam(group_name)
+ return group_info
+
+
 def add_user_to_group(username, group):
 """Add a user to a group"""
 cmd = [
@@ -103,7 +140,7 @@ def rsync(from_path, to_path, flags='-r', options=None):
 cmd.append(from_path)
 cmd.append(to_path)
 log(" ".join(cmd))
- return subprocess.check_output(cmd).strip()
+ return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
 def symlink(source, destination):
@@ -118,7 +155,7 @@ def symlink(source, destination):
 subprocess.check_call(cmd)
 
 
-def mkdir(path, owner='root', group='root', perms=0555, force=False):
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
 """Create a directory"""
 log("Making dir {} {}:{} {:o}".format(path, owner, group,
 perms))
@@ -134,7 +171,7 @@ def mkdir(path, owner='root', group='root', perms=0555, force=False):
 os.chown(realpath, uid, gid)
 
 
-def write_file(path, content, owner='root', group='root', perms=0444):
+def write_file(path, content, owner='root', group='root', perms=0o444):
 """Create or overwrite a file with the contents of a string"""
 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
 uid = pwd.getpwnam(owner).pw_uid
@@ -165,7 +202,7 @@ def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
 cmd_args.extend([device, mountpoint])
 try:
 subprocess.check_output(cmd_args)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
 return False
 
@@ -179,7 +216,7 @@ def umount(mountpoint, persist=False):
 cmd_args = ['umount', mountpoint]
 try:
 subprocess.check_output(cmd_args)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
 return False
 
@@ -197,17 +234,42 @@ def mounts():
 return system_mounts
 
 
-def file_hash(path):
- """Generate a md5 hash of the contents of 'path' or None if not found """
+def file_hash(path, hash_type='md5'):
+ """
+ Generate a hash checksum of the contents of 'path' or None if not found.
+
+ :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+ """
 if os.path.exists(path):
- h = hashlib.md5()
- with open(path, 'r') as source:
- h.update(source.read()) # IGNORE:E1101 - it does have update
+ h = getattr(hashlib, hash_type)()
+ with open(path, 'rb') as source:
+ h.update(source.read())
 return h.hexdigest()
 else:
 return None
 
 
+def check_hash(path, checksum, hash_type='md5'):
+ """
+ Validate a file using a cryptographic checksum.
+
+ :param str checksum: Value of the checksum used to validate the file.
+ :param str hash_type: Hash algorithm used to generate `checksum`.
+ Can be any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+ :raises ChecksumError: If the file fails the checksum
+
+ """
+ actual_checksum = file_hash(path, hash_type)
+ if checksum != actual_checksum:
+ raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
+
+
+class ChecksumError(ValueError):
+ pass
+
+
 def restart_on_change(restart_map, stopstart=False):
 """Restart services based on configuration files changing
 
@@ -260,7 +322,7 @@ def pwgen(length=None):
 if length is None:
 length = random.choice(range(35, 45))
 alphanumeric_chars = [
- l for l in (string.letters + string.digits)
+ l for l in (string.ascii_letters + string.digits)
 if l not in 'l0QD1vAEIOUaeiou']
 random_chars = [
 random.choice(alphanumeric_chars) for _ in range(length)]
@@ -269,18 +331,24 @@ def pwgen(length=None):
 
 def list_nics(nic_type):
 '''Return a list of nics of given type(s)'''
- if isinstance(nic_type, basestring):
+ if isinstance(nic_type, six.string_types):
 int_types = [nic_type]
 else:
 int_types = nic_type
 interfaces = []
 for int_type in int_types:
 cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
- ip_output = subprocess.check_output(cmd).split('\n')
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
 ip_output = (line for line in ip_output if line)
 for line in ip_output:
 if line.split()[1].startswith(int_type):
- interfaces.append(line.split()[1].replace(":", ""))
+ matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
+ if matched:
+ interface = matched.groups()[0]
+ else:
+ interface = line.split()[1].replace(":", "")
+ interfaces.append(interface)
+
 return interfaces
 
 
@@ -292,7 +360,7 @@ def set_nic_mtu(nic, mtu):
 
 def get_nic_mtu(nic):
 cmd = ['ip', 'addr', 'show', nic]
- ip_output = subprocess.check_output(cmd).split('\n')
+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
 mtu = ""
 for line in ip_output:
 words = line.split()
@@ -303,7 +371,7 @@ def get_nic_mtu(nic):
 
 def get_nic_hwaddr(nic):
 cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
- ip_output = subprocess.check_output(cmd)
+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
 hwaddr = ""
 words = ip_output.split()
 if 'link/ether' in words:
@@ -321,7 +389,28 @@ def cmp_pkgrevno(package, revno, pkgcache=None):
 '''
 import apt_pkg
 if not pkgcache:
- apt_pkg.init()
- pkgcache = apt_pkg.Cache()
+ from charmhelpers.fetch import apt_cache
+ pkgcache = apt_cache()
 pkg = pkgcache[package]
 return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
+
+
+@contextmanager
+def chdir(d):
+ cur = os.getcwd()
+ try:
+ yield os.chdir(d)
+ finally:
+ os.chdir(cur)
+
+
+def chownr(path, owner, group):
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+
+ for root, dirs, files in os.walk(path):
+ for name in dirs + files:
+ full = os.path.join(root, name)
+ broken_symlink = os.path.lexists(full) and not os.path.exists(full)
+ if not broken_symlink:
+ os.chown(full, uid, gid)
diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py
new file mode 100644
index 0000000..69dde79
--- /dev/null
+++ b/hooks/charmhelpers/core/services/__init__.py
@@ -0,0 +1,2 @@
+from .base import * # NOQA
+from .helpers import * # NOQA
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
new file mode 100644
index 0000000..87ecb13
--- /dev/null
+++ b/hooks/charmhelpers/core/services/base.py
@@ -0,0 +1,313 @@
+import os
+import re
+import json
+from collections import Iterable
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+__all__ = ['ServiceManager', 'ManagerCallback',
+ 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports',
+ 'service_restart', 'service_stop']
+
+
+class ServiceManager(object):
+ def __init__(self, services=None):
+ """
+ Register a list of services, given their definitions.
+
+ Service definitions are dicts in the following formats (all keys except
+ 'service' are optional)::
+
+ {
+ "service": <service name>,
+ "required_data": <list of required data contexts>,
+ "provided_data": <list of provided data contexts>,
+ "data_ready": <one or more callbacks>,
+ "data_lost": <one or more callbacks>,
+ "start": <one or more callbacks>,
+ "stop": <one or more callbacks>,
+ "ports": <list of ports to manage>,
+ }
+
+ The 'required_data' list should contain dicts of required data (or
+ dependency managers that act like dicts and know how to collect the data).
+ Only when all items in the 'required_data' list are populated are the list
+ of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more
+ information.
+
+ The 'provided_data' list should contain relation data providers, most likely
+ a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`,
+ that will indicate a set of data to set on a given relation.
+
+ The 'data_ready' value should be either a single callback, or a list of
+ callbacks, to be called when all items in 'required_data' pass `is_ready()`.
+ Each callback will be called with the service name as the only parameter.
+ After all of the 'data_ready' callbacks are called, the 'start' callbacks
+ are fired.
+
+ The 'data_lost' value should be either a single callback, or a list of
+ callbacks, to be called when a 'required_data' item no longer passes
+ `is_ready()`. Each callback will be called with the service name as the
+ only parameter. After all of the 'data_lost' callbacks are called,
+ the 'stop' callbacks are fired.
+
+ The 'start' value should be either a single callback, or a list of
+ callbacks, to be called when starting the service, after the 'data_ready'
+ callbacks are complete. Each callback will be called with the service
+ name as the only parameter. This defaults to
+ `[host.service_start, services.open_ports]`.
+
+ The 'stop' value should be either a single callback, or a list of
+ callbacks, to be called when stopping the service. If the service is
+ being stopped because it no longer has all of its 'required_data', this
+ will be called after all of the 'data_lost' callbacks are complete.
+ Each callback will be called with the service name as the only parameter.
+ This defaults to `[services.close_ports, host.service_stop]`.
+
+ The 'ports' value should be a list of ports to manage. The default
+ 'start' handler will open the ports after the service is started,
+ and the default 'stop' handler will close the ports prior to stopping
+ the service.
+
+
+ Examples:
+
+ The following registers an Upstart service called bingod that depends on
+ a mongodb relation and which runs a custom `db_migrate` function prior to
+ restarting the service, and a Runit service called spadesd::
+
+ manager = services.ServiceManager([
+ {
+ 'service': 'bingod',
+ 'ports': [80, 443],
+ 'required_data': [MongoRelation(), config(), {'my': 'data'}],
+ 'data_ready': [
+ services.template(source='bingod.conf'),
+ services.template(source='bingod.ini',
+ target='/etc/bingod.ini',
+ owner='bingo', perms=0400),
+ ],
+ },
+ {
+ 'service': 'spadesd',
+ 'data_ready': services.template(source='spadesd_run.j2',
+ target='/etc/sv/spadesd/run',
+ perms=0555),
+ 'start': runit_start,
+ 'stop': runit_stop,
+ },
+ ])
+ manager.manage()
+ """
+ self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
+ self._ready = None
+ self.services = {}
+ for service in services or []:
+ service_name = service['service']
+ self.services[service_name] = service
+
+ def manage(self):
+ """
+ Handle the current hook by doing The Right Thing with the registered services.
+ """
+ hook_name = hookenv.hook_name()
+ if hook_name == 'stop':
+ self.stop_services()
+ else:
+ self.provide_data()
+ self.reconfigure_services()
+ cfg = hookenv.config()
+ if cfg.implicit_save:
+ cfg.save()
+
+ def provide_data(self):
+ """
+ Set the relation data for each provider in the ``provided_data`` list.
+
+ A provider must have a `name` attribute, which indicates which relation
+ to set data on, and a `provide_data()` method, which returns a dict of
+ data to set.
+ """
+ hook_name = hookenv.hook_name()
+ for service in self.services.values():
+ for provider in service.get('provided_data', []):
+ if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
+ data = provider.provide_data()
+ _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
+ if _ready:
+ hookenv.relation_set(None, data)
+
+ def reconfigure_services(self, *service_names):
+ """
+ Update all files for one or more registered services, and,
+ if ready, optionally restart them.
+
+ If no service names are given, reconfigures all registered services.
+ """
+ for service_name in service_names or self.services.keys():
+ if self.is_ready(service_name):
+ self.fire_event('data_ready', service_name)
+ self.fire_event('start', service_name, default=[
+ service_restart,
+ manage_ports])
+ self.save_ready(service_name)
+ else:
+ if self.was_ready(service_name):
+ self.fire_event('data_lost', service_name)
+ self.fire_event('stop', service_name, default=[
+ manage_ports,
+ service_stop])
+ self.save_lost(service_name)
+
+ def stop_services(self, *service_names):
+ """
+ Stop one or more registered services, by name.
+
+ If no service names are given, stops all registered services.
+ """
+ for service_name in service_names or self.services.keys():
+ self.fire_event('stop', service_name, default=[
+ manage_ports,
+ service_stop])
+
+ def get_service(self, service_name):
+ """
+ Given the name of a registered service, return its service definition.
+ """
+ service = self.services.get(service_name)
+ if not service:
+ raise KeyError('Service not registered: %s' % service_name)
+ return service
+
+ def fire_event(self, event_name, service_name, default=None):
+ """
+ Fire a data_ready, data_lost, start, or stop event on a given service.
+ """
+ service = self.get_service(service_name)
+ callbacks = service.get(event_name, default)
+ if not callbacks:
+ return
+ if not isinstance(callbacks, Iterable):
+ callbacks = [callbacks]
+ for callback in callbacks:
+ if isinstance(callback, ManagerCallback):
+ callback(self, service_name, event_name)
+ else:
+ callback(service_name)
+
+ def is_ready(self, service_name):
+ """
+ Determine if a registered service is ready, by checking its 'required_data'.
+
+ A 'required_data' item can be any mapping type, and is considered ready
+ if `bool(item)` evaluates as True.
+ """
+ service = self.get_service(service_name)
+ reqs = service.get('required_data', [])
+ return all(bool(req) for req in reqs)
+
+ def _load_ready_file(self):
+ if self._ready is not None:
+ return
+ if os.path.exists(self._ready_file):
+ with open(self._ready_file) as fp:
+ self._ready = set(json.load(fp))
+ else:
+ self._ready = set()
+
+ def _save_ready_file(self):
+ if self._ready is None:
+ return
+ with open(self._ready_file, 'w') as fp:
+ json.dump(list(self._ready), fp)
+
+ def save_ready(self, service_name):
+ """
+ Save an indicator that the given service is now data_ready.
+ """
+ self._load_ready_file()
+ self._ready.add(service_name)
+ self._save_ready_file()
+
+ def save_lost(self, service_name):
+ """
+ Save an indicator that the given service is no longer data_ready.
+ """
+ self._load_ready_file()
+ self._ready.discard(service_name)
+ self._save_ready_file()
+
+ def was_ready(self, service_name):
+ """
+ Determine if the given service was previously data_ready.
+ """
+ self._load_ready_file()
+ return service_name in self._ready
+
+
+class ManagerCallback(object):
+ """
+ Special case of a callback that takes the `ServiceManager` instance
+ in addition to the service name.
+
+ Subclasses should implement `__call__` which should accept three parameters:
+
+ * `manager` The `ServiceManager` instance
+ * `service_name` The name of the service it's being triggered for
+ * `event_name` The name of the event that this callback is handling
+ """
+ def __call__(self, manager, service_name, event_name):
+ raise NotImplementedError()
+
+
+class PortManagerCallback(ManagerCallback):
+ """
+ Callback class that will open or close ports, for use as either
+ a start or stop action.
+ """
+ def __call__(self, manager, service_name, event_name):
+ service = manager.get_service(service_name)
+ new_ports = service.get('ports', [])
+ port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
+ if os.path.exists(port_file):
+ with open(port_file) as fp:
+ old_ports = fp.read().split(',')
+ for old_port in old_ports:
+ if bool(old_port):
+ old_port = int(old_port)
+ if old_port not in new_ports:
+ hookenv.close_port(old_port)
+ with open(port_file, 'w') as fp:
+ fp.write(','.join(str(port) for port in new_ports))
+ for port in new_ports:
+ if event_name == 'start':
+ hookenv.open_port(port)
+ elif event_name == 'stop':
+ hookenv.close_port(port)
+
+
+def service_stop(service_name):
+ """
+ Wrapper around host.service_stop to prevent spurious "unknown service"
+ messages in the logs.
+ """
+ if host.service_running(service_name):
+ host.service_stop(service_name)
+
+
+def service_restart(service_name):
+ """
+ Wrapper around host.service_restart to prevent spurious "unknown service"
+ messages in the logs.
+ """
+ if host.service_available(service_name):
+ if host.service_running(service_name):
+ host.service_restart(service_name)
+ else:
+ host.service_start(service_name)
+
+
+# Convenience aliases
+open_ports = close_ports = manage_ports = PortManagerCallback()
diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py
new file mode 100644
index 0000000..163a793
--- /dev/null
+++ b/hooks/charmhelpers/core/services/helpers.py
@@ -0,0 +1,243 @@
+import os
+import yaml
+from charmhelpers.core import hookenv
+from charmhelpers.core import templating
+
+from charmhelpers.core.services.base import ManagerCallback
+
+
+__all__ = ['RelationContext', 'TemplateCallback',
+ 'render_template', 'template']
+
+
+class RelationContext(dict):
+ """
+ Base class for a context generator that gets relation data from juju.
+
+ Subclasses must provide the attributes `name`, which is the name of the
+ interface of interest, `interface`, which is the type of the interface of
+ interest, and `required_keys`, which is the set of keys required for the
+ relation to be considered complete. The data for all interfaces matching
+ the `name` attribute that are complete will used to populate the dictionary
+ values (see `get_data`, below).
+
+ The generated context will be namespaced under the relation :attr:`name`,
+ to prevent potential naming conflicts.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = None
+ interface = None
+ required_keys = []
+
+ def __init__(self, name=None, additional_required_keys=None):
+ if name is not None:
+ self.name = name
+ if additional_required_keys is not None:
+ self.required_keys.extend(additional_required_keys)
+ self.get_data()
+
+ def __bool__(self):
+ """
+ Returns True if all of the required_keys are available.
+ """
+ return self.is_ready()
+
+ __nonzero__ = __bool__
+
+ def __repr__(self):
+ return super(RelationContext, self).__repr__()
+
+ def is_ready(self):
+ """
+ Returns True if all of the `required_keys` are available from any units.
+ """
+ ready = len(self.get(self.name, [])) > 0
+ if not ready:
+ hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG)
+ return ready
+
+ def _is_ready(self, unit_data):
+ """
+ Helper method that tests a set of relation data and returns True if
+ all of the `required_keys` are present.
+ """
+ return set(unit_data.keys()).issuperset(set(self.required_keys))
+
+ def get_data(self):
+ """
+ Retrieve the relation data for each unit involved in a relation and,
+ if complete, store it in a list under `self[self.name]`. This
+ is automatically called when the RelationContext is instantiated.
+
+ The units are sorted lexographically first by the service ID, then by
+ the unit ID. Thus, if an interface has two other services, 'db:1'
+ and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1',
+ and 'db:2' having one unit, 'mediawiki/0', all of which have a complete
+ set of data, the relation data for the units will be stored in the
+ order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'.
+
+ If you only care about a single unit on the relation, you can just
+ access it as `{{ interface[0]['key'] }}`. However, if you can at all
+ support multiple units on a relation, you should iterate over the list,
+ like::
+
+ {% for unit in interface -%}
+ {{ unit['key'] }}{% if not loop.last %},{% endif %}
+ {%- endfor %}
+
+ Note that since all sets of relation data from all related services and
+ units are in a single list, if you need to know which service or unit a
+ set of data came from, you'll need to extend this class to preserve
+ that information.
+ """
+ if not hookenv.relation_ids(self.name):
+ return
+
+ ns = self.setdefault(self.name, [])
+ for rid in sorted(hookenv.relation_ids(self.name)):
+ for unit in sorted(hookenv.related_units(rid)):
+ reldata = hookenv.relation_get(rid=rid, unit=unit)
+ if self._is_ready(reldata):
+ ns.append(reldata)
+
+ def provide_data(self):
+ """
+ Return data to be relation_set for this interface.
+ """
+ return {}
+
+
+class MysqlRelation(RelationContext):
+ """
+ Relation context for the `mysql` interface.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = 'db'
+ interface = 'mysql'
+ required_keys = ['host', 'user', 'password', 'database']
+
+
+class HttpRelation(RelationContext):
+ """
+ Relation context for the `http` interface.
+
+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
+ """
+ name = 'website'
+ interface = 'http'
+ required_keys = ['host', 'port']
+
+ def provide_data(self):
+ return {
+ 'host': hookenv.unit_get('private-address'),
+ 'port': 80,
+ }
+
+
+class RequiredConfig(dict):
+ """
+ Data context that loads config options with one or more mandatory options.
+
+ Once the required options have been changed from their default values, all
+ config options will be available, namespaced under `config` to prevent
+ potential naming conflicts (for example, between a config option and a
+ relation property).
+
+ :param list *args: List of options that must be changed from their default values.
+ """
+
+ def __init__(self, *args):
+ self.required_options = args
+ self['config'] = hookenv.config()
+ with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp:
+ self.config = yaml.load(fp).get('options', {})
+
+ def __bool__(self):
+ for option in self.required_options:
+ if option not in self['config']:
+ return False
+ current_value = self['config'][option]
+ default_value = self.config[option].get('default')
+ if current_value == default_value:
+ return False
+ if current_value in (None, '') and default_value in (None, ''):
+ return False
+ return True
+
+ def __nonzero__(self):
+ return self.__bool__()
+
+
+class StoredContext(dict):
+ """
+ A data context that always returns the data that it was first created with.
+
+ This is useful to do a one-time generation of things like passwords, that
+ will thereafter use the same value that was originally generated, instead
+ of generating a new value each time it is run.
+ """
+ def __init__(self, file_name, config_data):
+ """
+ If the file exists, populate `self` with the data from the file.
+ Otherwise, populate with the given data and persist it to the file.
+ """
+ if os.path.exists(file_name):
+ self.update(self.read_context(file_name))
+ else:
+ self.store_context(file_name, config_data)
+ self.update(config_data)
+
+ def store_context(self, file_name, config_data):
+ if not os.path.isabs(file_name):
+ file_name = os.path.join(hookenv.charm_dir(), file_name)
+ with open(file_name, 'w') as file_stream:
+ os.fchmod(file_stream.fileno(), 0o600)
+ yaml.dump(config_data, file_stream)
+
+ def read_context(self, file_name):
+ if not os.path.isabs(file_name):
+ file_name = os.path.join(hookenv.charm_dir(), file_name)
+ with open(file_name, 'r') as file_stream:
+ data = yaml.load(file_stream)
+ if not data:
+ raise OSError("%s is empty" % file_name)
+ return data
+
+
+class TemplateCallback(ManagerCallback):
+ """
+ Callback class that will render a Jinja2 template, for use as a ready
+ action.
+
+ :param str source: The template source file, relative to
+ `$CHARM_DIR/templates`
+
+ :param str target: The target to write the rendered template to
+ :param str owner: The owner of the rendered file
+ :param str group: The group of the rendered file
+ :param int perms: The permissions of the rendered file
+ """
+ def __init__(self, source, target,
+ owner='root', group='root', perms=0o444):
+ self.source = source
+ self.target = target
+ self.owner = owner
+ self.group = group
+ self.perms = perms
+
+ def __call__(self, manager, service_name, event_name):
+ service = manager.get_service(service_name)
+ context = {}
+ for ctx in service.get('required_data', []):
+ context.update(ctx)
+ templating.render(self.source, self.target, context,
+ self.owner, self.group, self.perms)
+
+
+# Convenience aliases for templates
+render_template = template = TemplateCallback
diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py
new file mode 100644
index 0000000..0f29963
--- /dev/null
+++ b/hooks/charmhelpers/core/sysctl.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
+
+import yaml
+
+from subprocess import check_call
+
+from charmhelpers.core.hookenv import (
+ log,
+ DEBUG,
+)
+
+
+def create(sysctl_dict, sysctl_file):
+ """Creates a sysctl.conf file from a YAML associative array
+
+ :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 }
+ :type sysctl_dict: dict
+ :param sysctl_file: path to the sysctl file to be saved
+ :type sysctl_file: str or unicode
+ :returns: None
+ """
+ sysctl_dict = yaml.load(sysctl_dict)
+
+ with open(sysctl_file, "w") as fd:
+ for key, value in sysctl_dict.items():
+ fd.write("{}={}\n".format(key, value))
+
+ log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict),
+ level=DEBUG)
+
+ check_call(["sysctl", "-p", sysctl_file])
diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py
new file mode 100644
index 0000000..83133fa
--- /dev/null
+++ b/hooks/charmhelpers/core/templating.py
@@ -0,0 +1,52 @@
+import os
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+def render(source, target, context, owner='root', group='root',
+ perms=0o444, templates_dir=None):
+ """
+ Render a template.
+
+ The `source` path, if not absolute, is relative to the `templates_dir`.
+
+ The `target` path should be absolute.
+
+ The context should be a dict containing the values to be replaced in the
+ template.
+
+ The `owner`, `group`, and `perms` options will be passed to `write_file`.
+
+ If omitted, `templates_dir` defaults to the `templates` folder in the charm.
+
+ Note: Using this requires python-jinja2; if it is not installed, calling
+ this will attempt to use charmhelpers.fetch.apt_install to install it.
+ """
+ try:
+ from jinja2 import FileSystemLoader, Environment, exceptions
+ except ImportError:
+ try:
+ from charmhelpers.fetch import apt_install
+ except ImportError:
+ hookenv.log('Could not import jinja2, and could not import '
+ 'charmhelpers.fetch to install it',
+ level=hookenv.ERROR)
+ raise
+ apt_install('python-jinja2', fatal=True)
+ from jinja2 import FileSystemLoader, Environment, exceptions
+
+ if templates_dir is None:
+ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+ loader = Environment(loader=FileSystemLoader(templates_dir))
+ try:
+ source = source
+ template = loader.get_template(source)
+ except exceptions.TemplateNotFound as e:
+ hookenv.log('Could not load template %s from %s.' %
+ (source, templates_dir),
+ level=hookenv.ERROR)
+ raise e
+ content = template.render(context)
+ host.mkdir(os.path.dirname(target))
+ host.write_file(target, content, owner, group, perms)
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 5be512c..0a126fc 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -1,13 +1,10 @@
 import importlib
+from tempfile import NamedTemporaryFile
 import time
 from yaml import safe_load
 from charmhelpers.core.host import (
 lsb_release
 )
-from urlparse import (
- urlparse,
- urlunparse,
-)
 import subprocess
 from charmhelpers.core.hookenv import (
 config,
@@ -15,6 +12,12 @@ from charmhelpers.core.hookenv import (
 )
 import os
 
+import six
+if six.PY3:
+ from urllib.parse import urlparse, urlunparse
+else:
+ from urlparse import urlparse, urlunparse
+
 
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@@ -71,6 +74,7 @@ CLOUD_ARCHIVE_POCKETS = {
 FETCH_HANDLERS = (
 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+ 'charmhelpers.fetch.giturl.GitUrlFetchHandler',
 )
 
 APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
@@ -116,14 +120,7 @@ class BaseFetchHandler(object):
 
 def filter_installed_packages(packages):
 """Returns a list of packages that require installation"""
- import apt_pkg
- apt_pkg.init()
-
- # Tell apt to build an in-memory cache to prevent race conditions (if
- # another process is already building the cache).
- apt_pkg.config.set("Dir::Cache::pkgcache", "")
-
- cache = apt_pkg.Cache()
+ cache = apt_cache()
 _pkgs = []
 for package in packages:
 try:
@@ -136,6 +133,16 @@ def filter_installed_packages(packages):
 return _pkgs
 
 
+def apt_cache(in_memory=True):
+ """Build and return an apt cache"""
+ import apt_pkg
+ apt_pkg.init()
+ if in_memory:
+ apt_pkg.config.set("Dir::Cache::pkgcache", "")
+ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+ return apt_pkg.Cache()
+
+
 def apt_install(packages, options=None, fatal=False):
 """Install one or more packages"""
 if options is None:
@@ -144,7 +151,7 @@ def apt_install(packages, options=None, fatal=False):
 cmd = ['apt-get', '--assume-yes']
 cmd.extend(options)
 cmd.append('install')
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -177,7 +184,7 @@ def apt_update(fatal=False):
 def apt_purge(packages, fatal=False):
 """Purge one or more packages"""
 cmd = ['apt-get', '--assume-yes', 'purge']
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -188,7 +195,7 @@ def apt_purge(packages, fatal=False):
 def apt_hold(packages, fatal=False):
 """Hold one or more packages"""
 cmd = ['apt-mark', 'hold']
- if isinstance(packages, basestring):
+ if isinstance(packages, six.string_types):
 cmd.append(packages)
 else:
 cmd.extend(packages)
@@ -201,6 +208,29 @@ def apt_hold(packages, fatal=False):
 
 
 def add_source(source, key=None):
+ """Add a package source to this system.
+
+ @param source: a URL or sources.list entry, as supported by
+ add-apt-repository(1). Examples::
+
+ ppa:charmers/example
+ deb https://stub:key@private.example.com/ubuntu trusty main
+
+ In addition:
+ 'proposed:' may be used to enable the standard 'proposed'
+ pocket for the release.
+ 'cloud:' may be used to activate official cloud archive pockets,
+ such as 'cloud:icehouse'
+ 'distro' may be used as a noop
+
+ @param key: A key to be added to the system's APT keyring and used
+ to verify the signatures on packages. Ideally, this should be an
+ ASCII format GPG public key including the block headers. A GPG key
+ id may also be used, but be aware that only insecure protocols are
+ available to retrieve the actual public key from a public keyserver
+ placing your Juju environment at risk. ppa and cloud archive keys
+ are securely added automtically, so sould not be provided.
+ """
 if source is None:
 log('Source is not present. Skipping')
 return
@@ -225,10 +255,25 @@ def add_source(source, key=None):
 release = lsb_release()['DISTRIB_CODENAME']
 with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
 apt.write(PROPOSED_POCKET.format(release))
+ elif source == 'distro':
+ pass
+ else:
+ log("Unknown source: {!r}".format(source))
+
 if key:
- subprocess.check_call(['apt-key', 'adv', '--keyserver',
- 'hkp://keyserver.ubuntu.com:80', '--recv',
- key])
+ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
+ with NamedTemporaryFile('w+') as key_file:
+ key_file.write(key)
+ key_file.flush()
+ key_file.seek(0)
+ subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
+ else:
+ # Note that hkp: is in no way a secure protocol. Using a
+ # GPG key id is pointless from a security POV unless you
+ # absolutely trust your network and DNS.
+ subprocess.check_call(['apt-key', 'adv', '--keyserver',
+ 'hkp://keyserver.ubuntu.com:80', '--recv',
+ key])
 
 
 def configure_sources(update=False,
@@ -238,7 +283,8 @@ def configure_sources(update=False,
 Configure multiple sources from charm configuration.
 
 The lists are encoded as yaml fragments in the configuration.
- The frament needs to be included as a string.
+ The frament needs to be included as a string. Sources and their
+ corresponding keys are of the types supported by add_source().
 
 Example config:
 install_sources: |
@@ -253,14 +299,14 @@ def configure_sources(update=False,
 sources = safe_load((config(sources_var) or '').strip()) or []
 keys = safe_load((config(keys_var) or '').strip()) or None
 
- if isinstance(sources, basestring):
+ if isinstance(sources, six.string_types):
 sources = [sources]
 
 if keys is None:
 for source in sources:
 add_source(source, None)
 else:
- if isinstance(keys, basestring):
+ if isinstance(keys, six.string_types):
 keys = [keys]
 
 if len(sources) != len(keys):
@@ -272,22 +318,35 @@ def configure_sources(update=False,
 apt_update(fatal=True)
 
 
-def install_remote(source):
+def install_remote(source, *args, **kwargs):
 """
 Install a file tree from a remote source
 
 The specified source should be a url of the form:
 scheme://[host]/path[#[option=value][&...]]
 
- Schemes supported are based on this modules submodules
- Options supported are submodule-specific"""
+ Schemes supported are based on this modules submodules.
+ Options supported are submodule-specific.
+ Additional arguments are passed through to the submodule.
+
+ For example::
+
+ dest = install_remote('http://example.com/archive.tgz',
+ checksum='deadbeef',
+ hash_type='sha1')
+
+ This will download `archive.tgz`, validate it using SHA1 and, if
+ the file is ok, extract it and return the directory in which it
+ was extracted. If the checksum fails, it will raise
+ :class:`charmhelpers.core.host.ChecksumError`.
+ """
 # We ONLY check for True here because can_handle may return a string
 # explaining why it can't handle a given source.
 handlers = [h for h in plugins() if h.can_handle(source) is True]
 installed_to = None
 for handler in handlers:
 try:
- installed_to = handler.install(source)
+ installed_to = handler.install(source, *args, **kwargs)
 except UnhandledSource:
 pass
 if not installed_to:
@@ -344,7 +403,7 @@ def _run_apt_command(cmd, fatal=False):
 while result is None or result == APT_NO_LOCK:
 try:
 result = subprocess.check_call(cmd, env=env)
- except subprocess.CalledProcessError, e:
+ except subprocess.CalledProcessError as e:
 retry_count = retry_count + 1
 if retry_count > APT_NO_LOCK_RETRY_COUNT:
 raise
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
index 87e7071..8a4624b 100644
--- a/hooks/charmhelpers/fetch/archiveurl.py
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -1,6 +1,23 @@
 import os
-import urllib2
-import urlparse
+import hashlib
+import re
+
+import six
+if six.PY3:
+ from urllib.request import (
+ build_opener, install_opener, urlopen, urlretrieve,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ )
+ from urllib.parse import urlparse, urlunparse, parse_qs
+ from urllib.error import URLError
+else:
+ from urllib import urlretrieve
+ from urllib2 import (
+ build_opener, install_opener, urlopen,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ URLError
+ )
+ from urlparse import urlparse, urlunparse, parse_qs
 
 from charmhelpers.fetch import (
 BaseFetchHandler,
@@ -10,11 +27,37 @@ from charmhelpers.payload.archive import (
 get_archive_handler,
 extract,
 )
-from charmhelpers.core.host import mkdir
+from charmhelpers.core.host import mkdir, check_hash
+
+
+def splituser(host):
+ '''urllib.splituser(), but six's support of this seems broken'''
+ _userprog = re.compile('^(.*)@(.*)$')
+ match = _userprog.match(host)
+ if match:
+ return match.group(1, 2)
+ return None, host
+
+
+def splitpasswd(user):
+ '''urllib.splitpasswd(), but six's support of this is missing'''
+ _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+ match = _passwdprog.match(user)
+ if match:
+ return match.group(1, 2)
+ return user, None
 
 
 class ArchiveUrlFetchHandler(BaseFetchHandler):
- """Handler for archives via generic URLs"""
+ """
+ Handler to download archive files from arbitrary URLs.
+
+ Can fetch from http, https, ftp, and file URLs.
+
+ Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
+
+ Installs the contents of the archive in $CHARM_DIR/fetched/.
+ """
 def can_handle(self, source):
 url_parts = self.parse_url(source)
 if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
@@ -24,22 +67,28 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
 return False
 
 def download(self, source, dest):
+ """
+ Download an archive file.
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local path location to download archive file to.
+ """
 # propogate all exceptions
 # URLError, OSError, etc
- proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+ proto, netloc, path, params, query, fragment = urlparse(source)
 if proto in ('http', 'https'):
- auth, barehost = urllib2.splituser(netloc)
+ auth, barehost = splituser(netloc)
 if auth is not None:
- source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
- username, password = urllib2.splitpasswd(auth)
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ source = urlunparse((proto, barehost, path, params, query, fragment))
+ username, password = splitpasswd(auth)
+ passman = HTTPPasswordMgrWithDefaultRealm()
 # Realm is set to None in add_password to force the username and password
 # to be used whatever the realm
 passman.add_password(None, source, username, password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(authhandler)
- urllib2.install_opener(opener)
- response = urllib2.urlopen(source)
+ authhandler = HTTPBasicAuthHandler(passman)
+ opener = build_opener(authhandler)
+ install_opener(opener)
+ response = urlopen(source)
 try:
 with open(dest, 'w') as dest_file:
 dest_file.write(response.read())
@@ -48,16 +97,49 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
 os.unlink(dest)
 raise e
 
- def install(self, source):
+ # Mandatory file validation via Sha1 or MD5 hashing.
+ def download_and_validate(self, url, hashsum, validate="sha1"):
+ tempfile, headers = urlretrieve(url)
+ check_hash(tempfile, hashsum, validate)
+ return tempfile
+
+ def install(self, source, dest=None, checksum=None, hash_type='sha1'):
+ """
+ Download and install an archive file, with optional checksum validation.
+
+ The checksum can also be given on the `source` URL's fragment.
+ For example::
+
+ handler.install('http://example.com/file.tgz#sha1=deadbeef')
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local destination path to install to. If not given,
+ installs to `$CHARM_DIR/archives/archive_file_name`.
+ :param str checksum: If given, validate the archive file after download.
+ :param str hash_type: Algorithm used to generate `checksum`.
+ Can be any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+
+ """
 url_parts = self.parse_url(source)
 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
 if not os.path.exists(dest_dir):
- mkdir(dest_dir, perms=0755)
+ mkdir(dest_dir, perms=0o755)
 dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
 try:
 self.download(source, dld_file)
- except urllib2.URLError as e:
+ except URLError as e:
 raise UnhandledSource(e.reason)
 except OSError as e:
 raise UnhandledSource(e.strerror)
- return extract(dld_file)
+ options = parse_qs(url_parts.fragment)
+ for key, value in options.items():
+ if not six.PY3:
+ algorithms = hashlib.algorithms
+ else:
+ algorithms = hashlib.algorithms_available
+ if key in algorithms:
+ check_hash(dld_file, value, key)
+ if checksum:
+ check_hash(dld_file, checksum, hash_type)
+ return extract(dld_file, dest)
diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py
index 0e580e4..8ef48f3 100644
--- a/hooks/charmhelpers/fetch/bzrurl.py
+++ b/hooks/charmhelpers/fetch/bzrurl.py
@@ -5,6 +5,10 @@ from charmhelpers.fetch import (
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+ raise ImportError('bzrlib does not support Python3')
+
 try:
 from bzrlib.branch import Branch
 except ImportError:
@@ -42,7 +46,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
 branch_name)
 if not os.path.exists(dest_dir):
- mkdir(dest_dir, perms=0755)
+ mkdir(dest_dir, perms=0o755)
 try:
 self.branch(source, dest_dir)
 except OSError as e:
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
new file mode 100644
index 0000000..f3aa282
--- /dev/null
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -0,0 +1,51 @@
+import os
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.core.host import mkdir
+
+import six
+if six.PY3:
+ raise ImportError('GitPython does not support Python 3')
+
+try:
+ from git import Repo
+except ImportError:
+ from charmhelpers.fetch import apt_install
+ apt_install("python-git")
+ from git import Repo
+
+
+class GitUrlFetchHandler(BaseFetchHandler):
+ """Handler for git branches via generic and github URLs"""
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ # TODO (mattyw) no support for ssh git@ yet
+ if url_parts.scheme not in ('http', 'https', 'git'):
+ return False
+ else:
+ return True
+
+ def clone(self, source, dest, branch):
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+
+ repo = Repo.clone_from(source, dest)
+ repo.git.checkout(branch)
+
+ def install(self, source, branch="master", dest=None):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
+ if not os.path.exists(dest_dir):
+ mkdir(dest_dir, perms=0o755)
+ try:
+ self.clone(source, dest_dir, branch)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir