Add charm-helper support.

9 files changed, 1127 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index ae1d5af..84be4d0 100644
--- a/Makefile
+++ b/Makefile
@@ -14,5 +14,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 
-unittest: 
+unittest:
	tests/10-unit.test
+
+sync:
+	@charm-helper-sync -c charm-helpers-sync.yaml
diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml
new file mode 100644
index 0000000..7fd4ca6
--- /dev/null
+++ b/charm-helpers-sync.yaml
@@ -0,0 +1,5 @@
+branch: lp:charm-helpers
+destination: hooks/charmhelpers
+include:
+ - core
+ - fetch
diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/hooks/charmhelpers/__init__.py
diff --git a/hooks/charmhelpers/core/__init__.py b/hooks/charmhelpers/core/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/hooks/charmhelpers/core/__init__.py
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
new file mode 100644
index 0000000..505c202
--- /dev/null
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -0,0 +1,401 @@
+"Interactions with the Juju environment"
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+# Charm Helpers Developers <juju@lists.ubuntu.com>
+
+import os
+import json
+import yaml
+import subprocess
+import sys
+import UserDict
+from subprocess import CalledProcessError
+
+CRITICAL = "CRITICAL"
+ERROR = "ERROR"
+WARNING = "WARNING"
+INFO = "INFO"
+DEBUG = "DEBUG"
+MARKER = object()
+
+cache = {}
+
+
+def cached(func):
+ """Cache return values for multiple executions of func + args
+
+ For example:
+
+ @cached
+ def unit_get(attribute):
+ pass
+
+ unit_get('test')
+
+ will cache the result of unit_get + 'test' for future calls.
+ """
+ def wrapper(*args, **kwargs):
+ global cache
+ key = str((func, args, kwargs))
+ try:
+ return cache[key]
+ except KeyError:
+ res = func(*args, **kwargs)
+ cache[key] = res
+ return res
+ return wrapper
+
+
+def flush(key):
+ """Flushes any entries from function cache where the
+ key is found in the function+args """
+ flush_list = []
+ for item in cache:
+ if key in item:
+ flush_list.append(item)
+ for item in flush_list:
+ del cache[item]
+
+
+def log(message, level=None):
+ """Write a message to the juju log"""
+ command = ['juju-log']
+ if level:
+ command += ['-l', level]
+ command += [message]
+ subprocess.call(command)
+
+
+class Serializable(UserDict.IterableUserDict):
+ """Wrapper, an object that can be serialized to yaml or json"""
+
+ def __init__(self, obj):
+ # wrap the object
+ UserDict.IterableUserDict.__init__(self)
+ self.data = obj
+
+ def __getattr__(self, attr):
+ # See if this object has attribute.
+ if attr in ("json", "yaml", "data"):
+ return self.__dict__[attr]
+ # Check for attribute in wrapped object.
+ got = getattr(self.data, attr, MARKER)
+ if got is not MARKER:
+ return got
+ # Proxy to the wrapped object via dict interface.
+ try:
+ return self.data[attr]
+ except KeyError:
+ raise AttributeError(attr)
+
+ def __getstate__(self):
+ # Pickle as a standard dictionary.
+ return self.data
+
+ def __setstate__(self, state):
+ # Unpickle into our wrapper.
+ self.data = state
+
+ def json(self):
+ """Serialize the object to json"""
+ return json.dumps(self.data)
+
+ def yaml(self):
+ """Serialize the object to yaml"""
+ return yaml.dump(self.data)
+
+
+def execution_environment():
+ """A convenient bundling of the current execution context"""
+ context = {}
+ context['conf'] = config()
+ if relation_id():
+ context['reltype'] = relation_type()
+ context['relid'] = relation_id()
+ context['rel'] = relation_get()
+ context['unit'] = local_unit()
+ context['rels'] = relations()
+ context['env'] = os.environ
+ return context
+
+
+def in_relation_hook():
+ """Determine whether we're running in a relation hook"""
+ return 'JUJU_RELATION' in os.environ
+
+
+def relation_type():
+ """The scope for the current relation hook"""
+ return os.environ.get('JUJU_RELATION', None)
+
+
+def relation_id():
+ """The relation ID for the current relation hook"""
+ return os.environ.get('JUJU_RELATION_ID', None)
+
+
+def local_unit():
+ """Local unit ID"""
+ return os.environ['JUJU_UNIT_NAME']
+
+
+def remote_unit():
+ """The remote unit for the current relation hook"""
+ return os.environ['JUJU_REMOTE_UNIT']
+
+
+def service_name():
+ """The name service group this unit belongs to"""
+ return local_unit().split('/')[0]
+
+
+def hook_name():
+ """The name of the currently executing hook"""
+ return os.path.basename(sys.argv[0])
+
+
+@cached
+def config(scope=None):
+ """Juju charm configuration"""
+ config_cmd_line = ['config-get']
+ if scope is not None:
+ config_cmd_line.append(scope)
+ config_cmd_line.append('--format=json')
+ try:
+ return json.loads(subprocess.check_output(config_cmd_line))
+ except ValueError:
+ return None
+
+
+@cached
+def relation_get(attribute=None, unit=None, rid=None):
+ """Get relation information"""
+ _args = ['relation-get', '--format=json']
+ if rid:
+ _args.append('-r')
+ _args.append(rid)
+ _args.append(attribute or '-')
+ if unit:
+ _args.append(unit)
+ try:
+ return json.loads(subprocess.check_output(_args))
+ except ValueError:
+ return None
+ except CalledProcessError, e:
+ if e.returncode == 2:
+ return None
+ raise
+
+
+def relation_set(relation_id=None, relation_settings={}, **kwargs):
+ """Set relation information for the current unit"""
+ relation_cmd_line = ['relation-set']
+ if relation_id is not None:
+ relation_cmd_line.extend(('-r', relation_id))
+ for k, v in (relation_settings.items() + kwargs.items()):
+ if v is None:
+ relation_cmd_line.append('{}='.format(k))
+ else:
+ relation_cmd_line.append('{}={}'.format(k, v))
+ subprocess.check_call(relation_cmd_line)
+ # Flush cache of any relation-gets for local unit
+ flush(local_unit())
+
+
+@cached
+def relation_ids(reltype=None):
+ """A list of relation_ids"""
+ reltype = reltype or relation_type()
+ relid_cmd_line = ['relation-ids', '--format=json']
+ if reltype is not None:
+ relid_cmd_line.append(reltype)
+ return json.loads(subprocess.check_output(relid_cmd_line)) or []
+ return []
+
+
+@cached
+def related_units(relid=None):
+ """A list of related units"""
+ relid = relid or relation_id()
+ units_cmd_line = ['relation-list', '--format=json']
+ if relid is not None:
+ units_cmd_line.extend(('-r', relid))
+ return json.loads(subprocess.check_output(units_cmd_line)) or []
+
+
+@cached
+def relation_for_unit(unit=None, rid=None):
+ """Get the json represenation of a unit's relation"""
+ unit = unit or remote_unit()
+ relation = relation_get(unit=unit, rid=rid)
+ for key in relation:
+ if key.endswith('-list'):
+ relation[key] = relation[key].split()
+ relation['__unit__'] = unit
+ return relation
+
+
+@cached
+def relations_for_id(relid=None):
+ """Get relations of a specific relation ID"""
+ relation_data = []
+ relid = relid or relation_ids()
+ for unit in related_units(relid):
+ unit_data = relation_for_unit(unit, relid)
+ unit_data['__relid__'] = relid
+ relation_data.append(unit_data)
+ return relation_data
+
+
+@cached
+def relations_of_type(reltype=None):
+ """Get relations of a specific type"""
+ relation_data = []
+ reltype = reltype or relation_type()
+ for relid in relation_ids(reltype):
+ for relation in relations_for_id(relid):
+ relation['__relid__'] = relid
+ relation_data.append(relation)
+ return relation_data
+
+
+@cached
+def relation_types():
+ """Get a list of relation types supported by this charm"""
+ charmdir = os.environ.get('CHARM_DIR', '')
+ mdf = open(os.path.join(charmdir, 'metadata.yaml'))
+ md = yaml.safe_load(mdf)
+ rel_types = []
+ for key in ('provides', 'requires', 'peers'):
+ section = md.get(key)
+ if section:
+ rel_types.extend(section.keys())
+ mdf.close()
+ return rel_types
+
+
+@cached
+def relations():
+ """Get a nested dictionary of relation data for all related units"""
+ rels = {}
+ for reltype in relation_types():
+ relids = {}
+ for relid in relation_ids(reltype):
+ units = {local_unit(): relation_get(unit=local_unit(), rid=relid)}
+ for unit in related_units(relid):
+ reldata = relation_get(unit=unit, rid=relid)
+ units[unit] = reldata
+ relids[relid] = units
+ rels[reltype] = relids
+ return rels
+
+
+@cached
+def is_relation_made(relation, keys='private-address'):
+ '''
+ Determine whether a relation is established by checking for
+ presence of key(s). If a list of keys is provided, they
+ must all be present for the relation to be identified as made
+ '''
+ if isinstance(keys, str):
+ keys = [keys]
+ for r_id in relation_ids(relation):
+ for unit in related_units(r_id):
+ context = {}
+ for k in keys:
+ context[k] = relation_get(k, rid=r_id,
+ unit=unit)
+ if None not in context.values():
+ return True
+ return False
+
+
+def open_port(port, protocol="TCP"):
+ """Open a service network port"""
+ _args = ['open-port']
+ _args.append('{}/{}'.format(port, protocol))
+ subprocess.check_call(_args)
+
+
+def close_port(port, protocol="TCP"):
+ """Close a service network port"""
+ _args = ['close-port']
+ _args.append('{}/{}'.format(port, protocol))
+ subprocess.check_call(_args)
+
+
+@cached
+def unit_get(attribute):
+ """Get the unit ID for the remote unit"""
+ _args = ['unit-get', '--format=json', attribute]
+ try:
+ return json.loads(subprocess.check_output(_args))
+ except ValueError:
+ return None
+
+
+def unit_private_ip():
+ """Get this unit's private IP address"""
+ return unit_get('private-address')
+
+
+class UnregisteredHookError(Exception):
+ """Raised when an undefined hook is called"""
+ pass
+
+
+class Hooks(object):
+ """A convenient handler for hook functions.
+
+ Example:
+ hooks = Hooks()
+
+ # register a hook, taking its name from the function name
+ @hooks.hook()
+ def install():
+ ...
+
+ # register a hook, providing a custom hook name
+ @hooks.hook("config-changed")
+ def config_changed():
+ ...
+
+ if __name__ == "__main__":
+ # execute a hook based on the name the program is called by
+ hooks.execute(sys.argv)
+ """
+
+ def __init__(self):
+ super(Hooks, self).__init__()
+ self._hooks = {}
+
+ def register(self, name, function):
+ """Register a hook"""
+ self._hooks[name] = function
+
+ def execute(self, args):
+ """Execute a registered hook based on args[0]"""
+ hook_name = os.path.basename(args[0])
+ if hook_name in self._hooks:
+ self._hooks[hook_name]()
+ else:
+ raise UnregisteredHookError(hook_name)
+
+ def hook(self, *hook_names):
+ """Decorator, registering them as hooks"""
+ def wrapper(decorated):
+ for hook_name in hook_names:
+ self.register(hook_name, decorated)
+ else:
+ self.register(decorated.__name__, decorated)
+ if '_' in decorated.__name__:
+ self.register(
+ decorated.__name__.replace('_', '-'), decorated)
+ return decorated
+ return wrapper
+
+
+def charm_dir():
+ """Return the root directory of the current charm"""
+ return os.environ.get('CHARM_DIR')
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
new file mode 100644
index 0000000..cfd2684
--- /dev/null
+++ b/hooks/charmhelpers/core/host.py
@@ -0,0 +1,297 @@
+"""Tools for working with the host system"""
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+# Nick Moffitt <nick.moffitt@canonical.com>
+# Matthew Wedgwood <matthew.wedgwood@canonical.com>
+
+import os
+import pwd
+import grp
+import random
+import string
+import subprocess
+import hashlib
+
+from collections import OrderedDict
+
+from hookenv import log
+
+
+def service_start(service_name):
+ """Start a system service"""
+ return service('start', service_name)
+
+
+def service_stop(service_name):
+ """Stop a system service"""
+ return service('stop', service_name)
+
+
+def service_restart(service_name):
+ """Restart a system service"""
+ return service('restart', service_name)
+
+
+def service_reload(service_name, restart_on_failure=False):
+ """Reload a system service, optionally falling back to restart if reload fails"""
+ service_result = service('reload', service_name)
+ if not service_result and restart_on_failure:
+ service_result = service('restart', service_name)
+ return service_result
+
+
+def service(action, service_name):
+ """Control a system service"""
+ cmd = ['service', service_name, action]
+ return subprocess.call(cmd) == 0
+
+
+def service_running(service):
+ """Determine whether a system service is running"""
+ try:
+ output = subprocess.check_output(['service', service, 'status'])
+ except subprocess.CalledProcessError:
+ return False
+ else:
+ if ("start/running" in output or "is running" in output):
+ return True
+ else:
+ return False
+
+
+def adduser(username, password=None, shell='/bin/bash', system_user=False):
+ """Add a user to the system"""
+ try:
+ user_info = pwd.getpwnam(username)
+ log('user {0} already exists!'.format(username))
+ except KeyError:
+ log('creating user {0}'.format(username))
+ cmd = ['useradd']
+ if system_user or password is None:
+ cmd.append('--system')
+ else:
+ cmd.extend([
+ '--create-home',
+ '--shell', shell,
+ '--password', password,
+ ])
+ cmd.append(username)
+ subprocess.check_call(cmd)
+ user_info = pwd.getpwnam(username)
+ return user_info
+
+
+def add_user_to_group(username, group):
+ """Add a user to a group"""
+ cmd = [
+ 'gpasswd', '-a',
+ username,
+ group
+ ]
+ log("Adding user {} to group {}".format(username, group))
+ subprocess.check_call(cmd)
+
+
+def rsync(from_path, to_path, flags='-r', options=None):
+ """Replicate the contents of a path"""
+ options = options or ['--delete', '--executability']
+ cmd = ['/usr/bin/rsync', flags]
+ cmd.extend(options)
+ cmd.append(from_path)
+ cmd.append(to_path)
+ log(" ".join(cmd))
+ return subprocess.check_output(cmd).strip()
+
+
+def symlink(source, destination):
+ """Create a symbolic link"""
+ log("Symlinking {} as {}".format(source, destination))
+ cmd = [
+ 'ln',
+ '-sf',
+ source,
+ destination,
+ ]
+ subprocess.check_call(cmd)
+
+
+def mkdir(path, owner='root', group='root', perms=0555, force=False):
+ """Create a directory"""
+ log("Making dir {} {}:{} {:o}".format(path, owner, group,
+ perms))
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ realpath = os.path.abspath(path)
+ if os.path.exists(realpath):
+ if force and not os.path.isdir(realpath):
+ log("Removing non-directory file {} prior to mkdir()".format(path))
+ os.unlink(realpath)
+ else:
+ os.makedirs(realpath, perms)
+ os.chown(realpath, uid, gid)
+
+
+def write_file(path, content, owner='root', group='root', perms=0444):
+ """Create or overwrite a file with the contents of a string"""
+ log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
+ uid = pwd.getpwnam(owner).pw_uid
+ gid = grp.getgrnam(group).gr_gid
+ with open(path, 'w') as target:
+ os.fchown(target.fileno(), uid, gid)
+ os.fchmod(target.fileno(), perms)
+ target.write(content)
+
+
+def mount(device, mountpoint, options=None, persist=False):
+ """Mount a filesystem at a particular mountpoint"""
+ cmd_args = ['mount']
+ if options is not None:
+ cmd_args.extend(['-o', options])
+ cmd_args.extend([device, mountpoint])
+ try:
+ subprocess.check_output(cmd_args)
+ except subprocess.CalledProcessError, e:
+ log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
+ return False
+ if persist:
+ # TODO: update fstab
+ pass
+ return True
+
+
+def umount(mountpoint, persist=False):
+ """Unmount a filesystem"""
+ cmd_args = ['umount', mountpoint]
+ try:
+ subprocess.check_output(cmd_args)
+ except subprocess.CalledProcessError, e:
+ log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+ return False
+ if persist:
+ # TODO: update fstab
+ pass
+ return True
+
+
+def mounts():
+ """Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
+ with open('/proc/mounts') as f:
+ # [['/mount/point','/dev/path'],[...]]
+ system_mounts = [m[1::-1] for m in [l.strip().split()
+ for l in f.readlines()]]
+ return system_mounts
+
+
+def file_hash(path):
+ """Generate a md5 hash of the contents of 'path' or None if not found """
+ if os.path.exists(path):
+ h = hashlib.md5()
+ with open(path, 'r') as source:
+ h.update(source.read()) # IGNORE:E1101 - it does have update
+ return h.hexdigest()
+ else:
+ return None
+
+
+def restart_on_change(restart_map, stopstart=False):
+ """Restart services based on configuration files changing
+
+ This function is used a decorator, for example
+
+ @restart_on_change({
+ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+ })
+ def ceph_client_changed():
+ ...
+
+ In this example, the cinder-api and cinder-volume services
+ would be restarted if /etc/ceph/ceph.conf is changed by the
+ ceph_client_changed function.
+ """
+ def wrap(f):
+ def wrapped_f(*args):
+ checksums = {}
+ for path in restart_map:
+ checksums[path] = file_hash(path)
+ f(*args)
+ restarts = []
+ for path in restart_map:
+ if checksums[path] != file_hash(path):
+ restarts += restart_map[path]
+ services_list = list(OrderedDict.fromkeys(restarts))
+ if not stopstart:
+ for service_name in services_list:
+ service('restart', service_name)
+ else:
+ for action in ['stop', 'start']:
+ for service_name in services_list:
+ service(action, service_name)
+ return wrapped_f
+ return wrap
+
+
+def lsb_release():
+ """Return /etc/lsb-release in a dict"""
+ d = {}
+ with open('/etc/lsb-release', 'r') as lsb:
+ for l in lsb:
+ k, v = l.split('=')
+ d[k.strip()] = v.strip()
+ return d
+
+
+def pwgen(length=None):
+ """Generate a random pasword."""
+ if length is None:
+ length = random.choice(range(35, 45))
+ alphanumeric_chars = [
+ l for l in (string.letters + string.digits)
+ if l not in 'l0QD1vAEIOUaeiou']
+ random_chars = [
+ random.choice(alphanumeric_chars) for _ in range(length)]
+ return(''.join(random_chars))
+
+
+def list_nics(nic_type):
+ '''Return a list of nics of given type(s)'''
+ if isinstance(nic_type, basestring):
+ int_types = [nic_type]
+ else:
+ int_types = nic_type
+ interfaces = []
+ for int_type in int_types:
+ cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+ ip_output = subprocess.check_output(cmd).split('\n')
+ ip_output = (line for line in ip_output if line)
+ for line in ip_output:
+ if line.split()[1].startswith(int_type):
+ interfaces.append(line.split()[1].replace(":", ""))
+ return interfaces
+
+
+def set_nic_mtu(nic, mtu):
+ '''Set MTU on a network interface'''
+ cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
+ subprocess.check_call(cmd)
+
+
+def get_nic_mtu(nic):
+ cmd = ['ip', 'addr', 'show', nic]
+ ip_output = subprocess.check_output(cmd).split('\n')
+ mtu = ""
+ for line in ip_output:
+ words = line.split()
+ if 'mtu' in words:
+ mtu = words[words.index("mtu") + 1]
+ return mtu
+
+
+def get_nic_hwaddr(nic):
+ cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
+ ip_output = subprocess.check_output(cmd)
+ hwaddr = ""
+ words = ip_output.split()
+ if 'link/ether' in words:
+ hwaddr = words[words.index('link/ether') + 1]
+ return hwaddr
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
new file mode 100644
index 0000000..97a1991
--- /dev/null
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -0,0 +1,308 @@
+import importlib
+from yaml import safe_load
+from charmhelpers.core.host import (
+ lsb_release
+)
+from urlparse import (
+ urlparse,
+ urlunparse,
+)
+import subprocess
+from charmhelpers.core.hookenv import (
+ config,
+ log,
+)
+import apt_pkg
+import os
+
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+PROPOSED_POCKET = """# Proposed
+deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
+"""
+CLOUD_ARCHIVE_POCKETS = {
+ # Folsom
+ 'folsom': 'precise-updates/folsom',
+ 'precise-folsom': 'precise-updates/folsom',
+ 'precise-folsom/updates': 'precise-updates/folsom',
+ 'precise-updates/folsom': 'precise-updates/folsom',
+ 'folsom/proposed': 'precise-proposed/folsom',
+ 'precise-folsom/proposed': 'precise-proposed/folsom',
+ 'precise-proposed/folsom': 'precise-proposed/folsom',
+ # Grizzly
+ 'grizzly': 'precise-updates/grizzly',
+ 'precise-grizzly': 'precise-updates/grizzly',
+ 'precise-grizzly/updates': 'precise-updates/grizzly',
+ 'precise-updates/grizzly': 'precise-updates/grizzly',
+ 'grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-proposed/grizzly': 'precise-proposed/grizzly',
+ # Havana
+ 'havana': 'precise-updates/havana',
+ 'precise-havana': 'precise-updates/havana',
+ 'precise-havana/updates': 'precise-updates/havana',
+ 'precise-updates/havana': 'precise-updates/havana',
+ 'havana/proposed': 'precise-proposed/havana',
+ 'precise-havana/proposed': 'precise-proposed/havana',
+ 'precise-proposed/havana': 'precise-proposed/havana',
+ # Icehouse
+ 'icehouse': 'precise-updates/icehouse',
+ 'precise-icehouse': 'precise-updates/icehouse',
+ 'precise-icehouse/updates': 'precise-updates/icehouse',
+ 'precise-updates/icehouse': 'precise-updates/icehouse',
+ 'icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-proposed/icehouse': 'precise-proposed/icehouse',
+}
+
+
+def filter_installed_packages(packages):
+ """Returns a list of packages that require installation"""
+ apt_pkg.init()
+ cache = apt_pkg.Cache()
+ _pkgs = []
+ for package in packages:
+ try:
+ p = cache[package]
+ p.current_ver or _pkgs.append(package)
+ except KeyError:
+ log('Package {} has no installation candidate.'.format(package),
+ level='WARNING')
+ _pkgs.append(package)
+ return _pkgs
+
+
+def apt_install(packages, options=None, fatal=False):
+ """Install one or more packages"""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ cmd.append('install')
+ if isinstance(packages, basestring):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Installing {} with options: {}".format(packages,
+ options))
+ env = os.environ.copy()
+ if 'DEBIAN_FRONTEND' not in env:
+ env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+ if fatal:
+ subprocess.check_call(cmd, env=env)
+ else:
+ subprocess.call(cmd, env=env)
+
+
+def apt_upgrade(options=None, fatal=False, dist=False):
+ """Upgrade all packages"""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ if dist:
+ cmd.append('dist-upgrade')
+ else:
+ cmd.append('upgrade')
+ log("Upgrading with options: {}".format(options))
+
+ env = os.environ.copy()
+ if 'DEBIAN_FRONTEND' not in env:
+ env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+ if fatal:
+ subprocess.check_call(cmd, env=env)
+ else:
+ subprocess.call(cmd, env=env)
+
+
+def apt_update(fatal=False):
+ """Update local apt cache"""
+ cmd = ['apt-get', 'update']
+ if fatal:
+ subprocess.check_call(cmd)
+ else:
+ subprocess.call(cmd)
+
+
+def apt_purge(packages, fatal=False):
+ """Purge one or more packages"""
+ cmd = ['apt-get', '--assume-yes', 'purge']
+ if isinstance(packages, basestring):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Purging {}".format(packages))
+ if fatal:
+ subprocess.check_call(cmd)
+ else:
+ subprocess.call(cmd)
+
+
+def apt_hold(packages, fatal=False):
+ """Hold one or more packages"""
+ cmd = ['apt-mark', 'hold']
+ if isinstance(packages, basestring):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Holding {}".format(packages))
+ if fatal:
+ subprocess.check_call(cmd)
+ else:
+ subprocess.call(cmd)
+
+
+def add_source(source, key=None):
+ if source is None:
+ log('Source is not present. Skipping')
+ return
+
+ if (source.startswith('ppa:') or
+ source.startswith('http') or
+ source.startswith('deb ') or
+ source.startswith('cloud-archive:')):
+ subprocess.check_call(['add-apt-repository', '--yes', source])
+ elif source.startswith('cloud:'):
+ apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
+ fatal=True)
+ pocket = source.split(':')[-1]
+ if pocket not in CLOUD_ARCHIVE_POCKETS:
+ raise SourceConfigError(
+ 'Unsupported cloud: source option %s' %
+ pocket)
+ actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+ with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+ apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+ elif source == 'proposed':
+ release = lsb_release()['DISTRIB_CODENAME']
+ with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+ apt.write(PROPOSED_POCKET.format(release))
+ if key:
+ subprocess.check_call(['apt-key', 'adv', '--keyserver',
+ 'keyserver.ubuntu.com', '--recv',
+ key])
+
+
+class SourceConfigError(Exception):
+ pass
+
+
+def configure_sources(update=False,
+ sources_var='install_sources',
+ keys_var='install_keys'):
+ """
+ Configure multiple sources from charm configuration
+
+ Example config:
+ install_sources:
+ - "ppa:foo"
+ - "http://example.com/repo precise main"
+ install_keys:
+ - null
+ - "a1b2c3d4"
+
+ Note that 'null' (a.k.a. None) should not be quoted.
+ """
+ sources = safe_load(config(sources_var))
+ keys = config(keys_var)
+ if keys is not None:
+ keys = safe_load(keys)
+ if isinstance(sources, basestring) and (
+ keys is None or isinstance(keys, basestring)):
+ add_source(sources, keys)
+ else:
+ if not len(sources) == len(keys):
+ msg = 'Install sources and keys lists are different lengths'
+ raise SourceConfigError(msg)
+ for src_num in range(len(sources)):
+ add_source(sources[src_num], keys[src_num])
+ if update:
+ apt_update(fatal=True)
+
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+ 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+ 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+
+class UnhandledSource(Exception):
+ pass
+
+
+def install_remote(source):
+ """
+ Install a file tree from a remote source
+
+ The specified source should be a url of the form:
+ scheme://[host]/path[#[option=value][&...]]
+
+ Schemes supported are based on this modules submodules
+ Options supported are submodule-specific"""
+ # We ONLY check for True here because can_handle may return a string
+ # explaining why it can't handle a given source.
+ handlers = [h for h in plugins() if h.can_handle(source) is True]
+ installed_to = None
+ for handler in handlers:
+ try:
+ installed_to = handler.install(source)
+ except UnhandledSource:
+ pass
+ if not installed_to:
+ raise UnhandledSource("No handler found for source {}".format(source))
+ return installed_to
+
+
+def install_from_config(config_var_name):
+ charm_config = config()
+ source = charm_config[config_var_name]
+ return install_remote(source)
+
+
+class BaseFetchHandler(object):
+
+ """Base class for FetchHandler implementations in fetch plugins"""
+
+ def can_handle(self, source):
+ """Returns True if the source can be handled. Otherwise returns
+ a string explaining why it cannot"""
+ return "Wrong source type"
+
+ def install(self, source):
+ """Try to download and unpack the source. Return the path to the
+ unpacked files or raise UnhandledSource."""
+ raise UnhandledSource("Wrong source type {}".format(source))
+
+ def parse_url(self, url):
+ return urlparse(url)
+
+ def base_url(self, url):
+ """Return url without querystring or fragment"""
+ parts = list(self.parse_url(url))
+ parts[4:] = ['' for i in parts[4:]]
+ return urlunparse(parts)
+
+
+def plugins(fetch_handlers=None):
+ if not fetch_handlers:
+ fetch_handlers = FETCH_HANDLERS
+ plugin_list = []
+ for handler_name in fetch_handlers:
+ package, classname = handler_name.rsplit('.', 1)
+ try:
+ handler_class = getattr(
+ importlib.import_module(package),
+ classname)
+ plugin_list.append(handler_class())
+ except (ImportError, AttributeError):
+ # Skip missing plugins so that they can be ommitted from
+ # installation if desired
+ log("FetchHandler {} not found, skipping plugin".format(
+ handler_name))
+ return plugin_list
diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py
new file mode 100644
index 0000000..87e7071
--- /dev/null
+++ b/hooks/charmhelpers/fetch/archiveurl.py
@@ -0,0 +1,63 @@
+import os
+import urllib2
+import urlparse
+
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.payload.archive import (
+ get_archive_handler,
+ extract,
+)
+from charmhelpers.core.host import mkdir
+
+
+class ArchiveUrlFetchHandler(BaseFetchHandler):
+ """Handler for archives via generic URLs"""
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+ return "Wrong source type"
+ if get_archive_handler(self.base_url(source)):
+ return True
+ return False
+
+ def download(self, source, dest):
+ # propogate all exceptions
+ # URLError, OSError, etc
+ proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+ if proto in ('http', 'https'):
+ auth, barehost = urllib2.splituser(netloc)
+ if auth is not None:
+ source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
+ username, password = urllib2.splitpasswd(auth)
+ passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ # Realm is set to None in add_password to force the username and password
+ # to be used whatever the realm
+ passman.add_password(None, source, username, password)
+ authhandler = urllib2.HTTPBasicAuthHandler(passman)
+ opener = urllib2.build_opener(authhandler)
+ urllib2.install_opener(opener)
+ response = urllib2.urlopen(source)
+ try:
+ with open(dest, 'w') as dest_file:
+ dest_file.write(response.read())
+ except Exception as e:
+ if os.path.isfile(dest):
+ os.unlink(dest)
+ raise e
+
+ def install(self, source):
+ url_parts = self.parse_url(source)
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+ if not os.path.exists(dest_dir):
+ mkdir(dest_dir, perms=0755)
+ dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
+ try:
+ self.download(source, dld_file)
+ except urllib2.URLError as e:
+ raise UnhandledSource(e.reason)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return extract(dld_file)
diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py
new file mode 100644
index 0000000..db5dd9a
--- /dev/null
+++ b/hooks/charmhelpers/fetch/bzrurl.py
@@ -0,0 +1,49 @@
+import os
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.core.host import mkdir
+
+try:
+ from bzrlib.branch import Branch
+except ImportError:
+ from charmhelpers.fetch import apt_install
+ apt_install("python-bzrlib")
+ from bzrlib.branch import Branch
+
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+ """Handler for bazaar branches via generic and lp URLs"""
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('bzr+ssh', 'lp'):
+ return False
+ else:
+ return True
+
+ def branch(self, source, dest):
+ url_parts = self.parse_url(source)
+ # If we use lp:branchname scheme we need to load plugins
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+ if url_parts.scheme == "lp":
+ from bzrlib.plugin import load_plugins
+ load_plugins()
+ try:
+ remote_branch = Branch.open(source)
+ remote_branch.bzrdir.sprout(dest).open_branch()
+ except Exception as e:
+ raise e
+
+ def install(self, source):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
+ if not os.path.exists(dest_dir):
+ mkdir(dest_dir, perms=0755)
+ try:
+ self.branch(source, dest_dir)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir