Juju Charms Collection
memcached package

Merge lp:~freyes/charms/trusty/memcached/py3-tox into lp:charms/trusty/memcached

Trusty Tahr (14.04)
py3-tox
Merge into trunk

Proposed by Felipe Reyes on 2016-09-16

Status:

Merged

Merged at revision:

Proposed branch:

lp:~freyes/charms/trusty/memcached/py3-tox

Merge into:

lp:charms/trusty/memcached

Diff against target:

4386 lines (+2276/-937)

45 files modified

.bzrignore (+1/-0)
.coveragerc (+6/-0)
charm-helpers.yaml (+1/-0)
hooks/charmhelpers/__init__.py (+11/-13)
hooks/charmhelpers/contrib/__init__.py (+11/-13)
hooks/charmhelpers/contrib/hahelpers/__init__.py (+11/-13)
hooks/charmhelpers/contrib/hahelpers/apache.py (+30/-17)
hooks/charmhelpers/contrib/hahelpers/cluster.py (+70/-23)
hooks/charmhelpers/contrib/network/__init__.py (+11/-13)
hooks/charmhelpers/contrib/network/ip.py (+90/-43)
hooks/charmhelpers/contrib/network/ovs/__init__.py (+17/-15)
hooks/charmhelpers/contrib/network/ufw.py (+16/-19)
hooks/charmhelpers/core/__init__.py (+11/-13)
hooks/charmhelpers/core/decorators.py (+11/-13)
hooks/charmhelpers/core/files.py (+43/-0)
hooks/charmhelpers/core/fstab.py (+11/-13)
hooks/charmhelpers/core/hookenv.py (+247/-27)
hooks/charmhelpers/core/host.py (+396/-130)
hooks/charmhelpers/core/host_factory/centos.py (+56/-0)
hooks/charmhelpers/core/host_factory/ubuntu.py (+56/-0)
hooks/charmhelpers/core/hugepage.py (+69/-0)
hooks/charmhelpers/core/kernel.py (+72/-0)
hooks/charmhelpers/core/kernel_factory/centos.py (+17/-0)
hooks/charmhelpers/core/kernel_factory/ubuntu.py (+13/-0)
hooks/charmhelpers/core/services/__init__.py (+11/-13)
hooks/charmhelpers/core/services/base.py (+11/-13)
hooks/charmhelpers/core/services/helpers.py (+41/-18)
hooks/charmhelpers/core/strutils.py (+41/-13)
hooks/charmhelpers/core/sysctl.py (+11/-13)
hooks/charmhelpers/core/templating.py (+40/-24)
hooks/charmhelpers/core/unitdata.py (+72/-31)
hooks/charmhelpers/fetch/__init__.py (+45/-296)
hooks/charmhelpers/fetch/archiveurl.py (+19/-15)
hooks/charmhelpers/fetch/bzrurl.py (+48/-50)
hooks/charmhelpers/fetch/centos.py (+171/-0)
hooks/charmhelpers/fetch/giturl.py (+34/-38)
hooks/charmhelpers/fetch/ubuntu.py (+313/-0)
hooks/charmhelpers/osplatform.py (+19/-0)
hooks/memcached_hooks.py (+1/-1)
setup.cfg (+5/-0)
test-requirements-py2.txt (+9/-0)
test-requirements-py3.txt (+7/-0)
tox.ini (+38/-0)
unit_tests/test_memcached_hooks.py (+62/-46)
unit_tests/test_utils.py (+1/-1)

To merge this branch:

bzr merge lp:~freyes/charms/trusty/memcached/py3-tox

Reviewer	Date Requested	Status
Jorge Niedbalski (community)	2016-09-16	Approve on 2016-09-20
charmers	2016-09-16	Pending
Review via email: mp+306002@code.launchpad.net

Description of the change

This patch enables the charm to use python3, with this change it's possible to deploy trusty or xenial.

This change was tested with juju 2.0 and juju 1.0

Revision history for this message

Jorge Niedbalski (niedbalski) wrote on 2016-09-20:

Hello,

I checked all the targets, including lint py27, py34, py35, all seems good.

Thanks for fixing @freyes

LGTM.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk

Subscribers

People subscribed via source and target branches

to all changes:

Felipe Reyes

Nobuto Murata

charmers

 === modified file '.bzrignore'
 --- .bzrignore	2014-12-09 21:43:34 +0000
 +++ .bzrignore	2016-09-16 19:19:06 +0000
@@ -3,3 +3,4 @@
  bin/
  .coverage
  .venv
++.tox
 === added file '.coveragerc'
 --- .coveragerc	1970-01-01 00:00:00 +0000
 +++ .coveragerc	2016-09-16 19:19:06 +0000
@@ -0,0 +1,6 @@
++[report]
++# Regexes for lines to exclude from consideration
++exclude_lines =
++ if __name__ == .__main__.:
++include=
++ hooks/memcached*
 === modified file 'charm-helpers.yaml'
 --- charm-helpers.yaml	2015-02-13 12:32:01 +0000
 +++ charm-helpers.yaml	2016-09-16 19:19:06 +0000
@@ -5,3 +5,4 @@
  - core
  - contrib.network
  - contrib.hahelpers
++ - osplatform
 === modified file 'hooks/charmhelpers/__init__.py'
 --- hooks/charmhelpers/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  # Bootstrap charm-helpers, installing its dependencies if necessary using
  # only standard libraries.
 === modified file 'hooks/charmhelpers/contrib/__init__.py'
 --- hooks/charmhelpers/contrib/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/contrib/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,15 +1,13 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
 === modified file 'hooks/charmhelpers/contrib/hahelpers/__init__.py'
 --- hooks/charmhelpers/contrib/hahelpers/__init__.py	2015-02-13 12:32:01 +0000
 +++ hooks/charmhelpers/contrib/hahelpers/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,15 +1,13 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
 === modified file 'hooks/charmhelpers/contrib/hahelpers/apache.py'
 --- hooks/charmhelpers/contrib/hahelpers/apache.py	2015-02-13 12:32:01 +0000
 +++ hooks/charmhelpers/contrib/hahelpers/apache.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
+ #
  # Copyright 2012 Canonical Ltd.
@@ -24,6 +22,7 @@
  # Adam Gandelman <adamg@ubuntu.com>
+ #
++import os
  import subprocess
  from charmhelpers.core.hookenv import (
@@ -74,9 +73,23 @@
  return ca_cert
++def retrieve_ca_cert(cert_file):
++ cert = None
++ if os.path.isfile(cert_file):
++ with open(cert_file, 'r') as crt:
++ cert = crt.read()
++ return cert
++
++
  def install_ca_cert(ca_cert):
  if ca_cert:
-- with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
-- 'w') as crt:
-- crt.write(ca_cert)
-- subprocess.check_call(['update-ca-certificates', '--fresh'])
++ cert_file = ('/usr/local/share/ca-certificates/'
++ 'keystone_juju_ca_cert.crt')
++ old_cert = retrieve_ca_cert(cert_file)
++ if old_cert and old_cert == ca_cert:
++ log("CA cert is the same as installed version", level=INFO)
++ else:
++ log("Installing new CA cert", level=INFO)
++ with open(cert_file, 'w') as crt:
++ crt.write(ca_cert)
++ subprocess.check_call(['update-ca-certificates', '--fresh'])
 === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
 --- hooks/charmhelpers/contrib/hahelpers/cluster.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
+ #
  # Copyright 2012 Canonical Ltd.
@@ -41,10 +39,11 @@
  relation_get,
  config as config_get,
  INFO,
-- ERROR,
++ DEBUG,
  WARNING,
  unit_get,
-- is_leader as juju_is_leader
++ is_leader as juju_is_leader,
++ status_set,
+ )
  from charmhelpers.core.decorators import (
  retry_on_exception,
@@ -60,6 +59,10 @@
  pass
++class HAIncorrectConfig(Exception):
++ pass
++
++
  class CRMResourceNotFound(Exception):
  pass
@@ -274,27 +277,71 @@
  Obtains all relevant configuration from charm configuration required
  for initiating a relation to hacluster:
-- ha-bindiface, ha-mcastport, vip
++ ha-bindiface, ha-mcastport, vip, os-internal-hostname,
++ os-admin-hostname, os-public-hostname, os-access-hostname
  param: exclude_keys: list of setting key(s) to be excluded.
  returns: dict: A dict containing settings keyed by setting name.
-- raises: HAIncompleteConfig if settings are missing.
++ raises: HAIncompleteConfig if settings are missing or incorrect.
  '''
-- settings = ['ha-bindiface', 'ha-mcastport', 'vip']
++ settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname',
++ 'os-admin-hostname', 'os-public-hostname', 'os-access-hostname']
  conf = {}
  for setting in settings:
  if exclude_keys and setting in exclude_keys:
  continue
  conf[setting] = config_get(setting)
-- missing = []
-- [missing.append(s) for s, v in six.iteritems(conf) if v is None]
-- if missing:
-- log('Insufficient config data to configure hacluster.', level=ERROR)
-- raise HAIncompleteConfig
++
++ if not valid_hacluster_config():
++ raise HAIncorrectConfig('Insufficient or incorrect config data to '
++ 'configure hacluster.')
  return conf
++def valid_hacluster_config():
++ '''
++ Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname
++ must be set.
++
++ Note: ha-bindiface and ha-macastport both have defaults and will always
++ be set. We only care that either vip or dns-ha is set.
++
++ :returns: boolean: valid config returns true.
++ raises: HAIncompatibileConfig if settings conflict.
++ raises: HAIncompleteConfig if settings are missing.
++ '''
++ vip = config_get('vip')
++ dns = config_get('dns-ha')
++ if not(bool(vip) ^ bool(dns)):
++ msg = ('HA: Either vip or dns-ha must be set but not both in order to '
++ 'use high availability')
++ status_set('blocked', msg)
++ raise HAIncorrectConfig(msg)
++
++ # If dns-ha then one of os-*-hostname must be set
++ if dns:
++ dns_settings = ['os-internal-hostname', 'os-admin-hostname',
++ 'os-public-hostname', 'os-access-hostname']
++ # At this point it is unknown if one or all of the possible
++ # network spaces are in HA. Validate at least one is set which is
++ # the minimum required.
++ for setting in dns_settings:
++ if config_get(setting):
++ log('DNS HA: At least one hostname is set {}: {}'
++ ''.format(setting, config_get(setting)),
++ level=DEBUG)
++ return True
++
++ msg = ('DNS HA: At least one os-*-hostname(s) must be set to use '
++ 'DNS HA')
++ status_set('blocked', msg)
++ raise HAIncompleteConfig(msg)
++
++ log('VIP HA: VIP is set {}'.format(vip), level=DEBUG)
++ return True
++
++
  def canonical_url(configs, vip_setting='vip'):
  '''
  Returns the correct HTTP URL to this host given the state of HTTPS
 === modified file 'hooks/charmhelpers/contrib/network/__init__.py'
 --- hooks/charmhelpers/contrib/network/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/contrib/network/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,15 +1,13 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
 === modified file 'hooks/charmhelpers/contrib/network/ip.py'
 --- hooks/charmhelpers/contrib/network/ip.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/contrib/network/ip.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import glob
  import re
@@ -23,7 +21,7 @@
  from functools import partial
  from charmhelpers.core.hookenv import unit_get
--from charmhelpers.fetch import apt_install
++from charmhelpers.fetch import apt_install, apt_update
  from charmhelpers.core.hookenv import (
  log,
  WARNING,
@@ -32,13 +30,15 @@
  try:
  import netifaces
  except ImportError:
-- apt_install('python-netifaces')
++ apt_update(fatal=True)
++ apt_install('python-netifaces', fatal=True)
  import netifaces
  try:
  import netaddr
  except ImportError:
-- apt_install('python-netaddr')
++ apt_update(fatal=True)
++ apt_install('python-netaddr', fatal=True)
  import netaddr
@@ -51,7 +51,7 @@
  def no_ip_found_error_out(network):
-- errmsg = ("No IP address found in network: %s" % network)
++ errmsg = ("No IP address found in network(s): %s" % network)
  raise ValueError(errmsg)
@@ -59,7 +59,7 @@
  """Get an IPv4 or IPv6 address within the network from the host.
  :param network (str): CIDR presentation format. For example,
-- '192.168.1.0/24'.
++ '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
  :param fallback (str): If no address is found, return fallback.
  :param fatal (boolean): If no address is found, fallback is not
  set and fatal is True then exit(1).
@@ -73,24 +73,26 @@
  else:
  return None
-- _validate_cidr(network)
-- network = netaddr.IPNetwork(network)
-- for iface in netifaces.interfaces():
-- addresses = netifaces.ifaddresses(iface)
-- if network.version == 4 and netifaces.AF_INET in addresses:
-- addr = addresses[netifaces.AF_INET][0]['addr']
-- netmask = addresses[netifaces.AF_INET][0]['netmask']
-- cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
-- if cidr in network:
-- return str(cidr.ip)
++ networks = network.split() or [network]
++ for network in networks:
++ _validate_cidr(network)
++ network = netaddr.IPNetwork(network)
++ for iface in netifaces.interfaces():
++ addresses = netifaces.ifaddresses(iface)
++ if network.version == 4 and netifaces.AF_INET in addresses:
++ addr = addresses[netifaces.AF_INET][0]['addr']
++ netmask = addresses[netifaces.AF_INET][0]['netmask']
++ cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
++ if cidr in network:
++ return str(cidr.ip)
-- if network.version == 6 and netifaces.AF_INET6 in addresses:
-- for addr in addresses[netifaces.AF_INET6]:
-- if not addr['addr'].startswith('fe80'):
-- cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
-- addr['netmask']))
-- if cidr in network:
-- return str(cidr.ip)
++ if network.version == 6 and netifaces.AF_INET6 in addresses:
++ for addr in addresses[netifaces.AF_INET6]:
++ if not addr['addr'].startswith('fe80'):
++ cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
++ addr['netmask']))
++ if cidr in network:
++ return str(cidr.ip)
  if fallback is not None:
  return fallback
@@ -187,6 +189,15 @@
  get_netmask_for_address = partial(_get_for_address, key='netmask')
++def resolve_network_cidr(ip_address):
++ '''
++ Resolves the full address cidr of an ip_address based on
++ configured network interfaces
++ '''
++ netmask = get_netmask_for_address(ip_address)
++ return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
++
++
  def format_ipv6_addr(address):
  """If address is IPv6, wrap it in '[]' otherwise return None.
@@ -201,7 +212,16 @@
  def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
  fatal=True, exc_list=None):
-- """Return the assigned IP address for a given interface, if any."""
++ """Return the assigned IP address for a given interface, if any.
++
++ :param iface: network interface on which address(es) are expected to
++ be found.
++ :param inet_type: inet address family
++ :param inc_aliases: include alias interfaces in search
++ :param fatal: if True, raise exception if address not found
++ :param exc_list: list of addresses to ignore
++ :return: list of ip addresses
++ """
  # Extract nic if passed /dev/ethX
  if '/' in iface:
  iface = iface.split('/')[-1]
@@ -302,6 +322,14 @@
  We currently only support scope global IPv6 addresses i.e. non-temporary
  addresses. If no global IPv6 address is found, return the first one found
  in the ipv6 address list.
++
++ :param iface: network interface on which ipv6 address(es) are expected to
++ be found.
++ :param inc_aliases: include alias interfaces in search
++ :param fatal: if True, raise exception if address not found
++ :param exc_list: list of addresses to ignore
++ :param dynamic_only: only recognise dynamic addresses
++ :return: list of ipv6 addresses
  """
  addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
  inc_aliases=inc_aliases, fatal=fatal,
@@ -323,7 +351,7 @@
  cmd = ['ip', 'addr', 'show', iface]
  out = subprocess.check_output(cmd).decode('UTF-8')
  if dynamic_only:
-- key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
++ key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
  else:
  key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
@@ -375,10 +403,10 @@
  Returns True if address is a valid IP address.
  """
  try:
-- # Test to see if already an IPv4 address
-- socket.inet_aton(address)
++ # Test to see if already an IPv4/IPv6 address
++ address = netaddr.IPAddress(address)
  return True
-- except socket.error:
++ except netaddr.AddrFormatError:
  return False
@@ -386,7 +414,7 @@
  try:
  import dns.resolver
  except ImportError:
-- apt_install('python-dnspython')
++ apt_install('python-dnspython', fatal=True)
  import dns.resolver
  if isinstance(address, dns.name.Name):
@@ -430,13 +458,17 @@
  try:
  import dns.reversename
  except ImportError:
-- apt_install("python-dnspython")
++ apt_install("python-dnspython", fatal=True)
  import dns.reversename
  rev = dns.reversename.from_address(address)
  result = ns_query(rev)
++
  if not result:
-- return None
++ try:
++ result = socket.gethostbyaddr(address)[0]
++ except:
++ return None
  else:
  result = address
@@ -448,3 +480,18 @@
  return result
  else:
  return result.split('.')[0]
++
++
++def port_has_listener(address, port):
++ """
++ Returns True if the address:port is open and being listened to,
++ else False.
++
++ @param address: an IP address or hostname
++ @param port: integer port
++
++ Note calls 'zc' via a subprocess shell
++ """
++ cmd = ['nc', '-z', address, str(port)]
++ result = subprocess.call(cmd)
++ return not(bool(result))
 === modified file 'hooks/charmhelpers/contrib/network/ovs/__init__.py'
 --- hooks/charmhelpers/contrib/network/ovs/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/contrib/network/ovs/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  ''' Helpers for interacting with OpenvSwitch '''
  import subprocess
@@ -25,10 +23,14 @@
+ )
--def add_bridge(name):
++def add_bridge(name, datapath_type=None):
  ''' Add the named bridge to openvswitch '''
  log('Creating bridge {}'.format(name))
-- subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name])
++ cmd = ["ovs-vsctl", "--", "--may-exist", "add-br", name]
++ if datapath_type is not None:
++ cmd += ['--', 'set', 'bridge', name,
++ 'datapath_type={}'.format(datapath_type)]
++ subprocess.check_call(cmd)
  def del_bridge(name):
 === modified file 'hooks/charmhelpers/contrib/network/ufw.py'
 --- hooks/charmhelpers/contrib/network/ufw.py	2015-07-06 21:30:20 +0000
 +++ hooks/charmhelpers/contrib/network/ufw.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  """
  This module contains helpers to add and remove ufw rules.
@@ -40,7 +38,9 @@
  import re
  import os
  import subprocess
++
  from charmhelpers.core import hookenv
++from charmhelpers.core.kernel import modprobe, is_module_loaded
  __author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
@@ -82,14 +82,11 @@
  # do we have IPv6 in the machine?
  if os.path.isdir('/proc/sys/net/ipv6'):
  # is ip6tables kernel module loaded?
-- lsmod = subprocess.check_output(['lsmod'], universal_newlines=True)
-- matches = re.findall('^ip6_tables[ ]+', lsmod, re.M)
-- if len(matches) == 0:
++ if not is_module_loaded('ip6_tables'):
  # ip6tables support isn't complete, let's try to load it
  try:
-- subprocess.check_output(['modprobe', 'ip6_tables'],
-- universal_newlines=True)
-- # great, we could load the module
++ modprobe('ip6_tables')
++ # great, we can load the module
  return True
  except subprocess.CalledProcessError as ex:
  hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
 === modified file 'hooks/charmhelpers/core/__init__.py'
 --- hooks/charmhelpers/core/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/core/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,15 +1,13 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
 === modified file 'hooks/charmhelpers/core/decorators.py'
 --- hooks/charmhelpers/core/decorators.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/core/decorators.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
+ #
  # Copyright 2014 Canonical Ltd.
 === added file 'hooks/charmhelpers/core/files.py'
 --- hooks/charmhelpers/core/files.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/files.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,43 @@
++#!/usr/bin/env python
++# -*- coding: utf-8 -*-
++
++# Copyright 2014-2015 Canonical Limited.
++#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
++
++__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
++
++import os
++import subprocess
++
++
++def sed(filename, before, after, flags='g'):
++ """
++ Search and replaces the given pattern on filename.
++
++ :param filename: relative or absolute file path.
++ :param before: expression to be replaced (see 'man sed')
++ :param after: expression to replace with (see 'man sed')
++ :param flags: sed-compatible regex flags in example, to make
++ the search and replace case insensitive, specify ``flags="i"``.
++ The ``g`` flag is always specified regardless, so you do not
++ need to remember to include it when overriding this parameter.
++ :returns: If the sed command exit code was zero then return,
++ otherwise raise CalledProcessError.
++ """
++ expression = r's/{0}/{1}/{2}'.format(before,
++ after, flags)
++
++ return subprocess.check_call(["sed", "-i", "-r", "-e",
++ expression,
++ os.path.expanduser(filename)])
 === modified file 'hooks/charmhelpers/core/fstab.py'
 --- hooks/charmhelpers/core/fstab.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/fstab.py	2016-09-16 19:19:06 +0000
@@ -3,19 +3,17 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import io
  import os
 === modified file 'hooks/charmhelpers/core/hookenv.py'
 --- hooks/charmhelpers/core/hookenv.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/hookenv.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  "Interactions with the Juju environment"
  # Copyright 2013 Canonical Ltd.
@@ -21,6 +19,7 @@
  # Charm Helpers Developers <juju@lists.ubuntu.com>
  from __future__ import print_function
++import copy
  from distutils.version import LooseVersion
  from functools import wraps
  import glob
@@ -73,6 +72,7 @@
  res = func(*args, **kwargs)
  cache[key] = res
  return res
++ wrapper._wrapped = func
  return wrapper
@@ -172,9 +172,19 @@
  return os.environ.get('JUJU_RELATION', None)
--def relation_id():
-- """The relation ID for the current relation hook"""
-- return os.environ.get('JUJU_RELATION_ID', None)
++@cached
++def relation_id(relation_name=None, service_or_unit=None):
++ """The relation ID for the current or a specified relation"""
++ if not relation_name and not service_or_unit:
++ return os.environ.get('JUJU_RELATION_ID', None)
++ elif relation_name and service_or_unit:
++ service_name = service_or_unit.split('/')[0]
++ for relid in relation_ids(relation_name):
++ remote_service = remote_service_name(relid)
++ if remote_service == service_name:
++ return relid
++ else:
++ raise ValueError('Must specify neither or both of relation_name and service_or_unit')
  def local_unit():
@@ -192,9 +202,20 @@
  return local_unit().split('/')[0]
++@cached
++def remote_service_name(relid=None):
++ """The remote service name for a given relation-id (or the current relation)"""
++ if relid is None:
++ unit = remote_unit()
++ else:
++ units = related_units(relid)
++ unit = units[0] if units else None
++ return unit.split('/')[0] if unit else None
++
++
  def hook_name():
  """The name of the currently executing hook"""
-- return os.path.basename(sys.argv[0])
++ return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
  class Config(dict):
@@ -263,7 +284,7 @@
  self.path = path or self.path
  with open(self.path) as f:
  self._prev_dict = json.load(f)
-- for k, v in self._prev_dict.items():
++ for k, v in copy.deepcopy(self._prev_dict).items():
  if k not in self:
  self[k] = v
@@ -468,6 +489,76 @@
  @cached
++def peer_relation_id():
++ '''Get the peers relation id if a peers relation has been joined, else None.'''
++ md = metadata()
++ section = md.get('peers')
++ if section:
++ for key in section:
++ relids = relation_ids(key)
++ if relids:
++ return relids[0]
++ return None
++
++
++@cached
++def relation_to_interface(relation_name):
++ """
++ Given the name of a relation, return the interface that relation uses.
++
++ :returns: The interface name, or ``None``.
++ """
++ return relation_to_role_and_interface(relation_name)[1]
++
++
++@cached
++def relation_to_role_and_interface(relation_name):
++ """
++ Given the name of a relation, return the role and the name of the interface
++ that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
++
++ :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
++ """
++ _metadata = metadata()
++ for role in ('provides', 'requires', 'peers'):
++ interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
++ if interface:
++ return role, interface
++ return None, None
++
++
++@cached
++def role_and_interface_to_relations(role, interface_name):
++ """
++ Given a role and interface name, return a list of relation names for the
++ current charm that use that interface under that role (where role is one
++ of ``provides``, ``requires``, or ``peers``).
++
++ :returns: A list of relation names.
++ """
++ _metadata = metadata()
++ results = []
++ for relation_name, relation in _metadata.get(role, {}).items():
++ if relation['interface'] == interface_name:
++ results.append(relation_name)
++ return results
++
++
++@cached
++def interface_to_relations(interface_name):
++ """
++ Given an interface, return a list of relation names for the current
++ charm that use that interface.
++
++ :returns: A list of relation names.
++ """
++ results = []
++ for role in ('provides', 'requires', 'peers'):
++ results.extend(role_and_interface_to_relations(role, interface_name))
++ return results
++
++
++@cached
  def charm_name():
  """Get the name of the current charm as is specified on metadata.yaml"""
  return metadata().get('name')
@@ -543,6 +634,38 @@
  return unit_get('private-address')
++@cached
++def storage_get(attribute=None, storage_id=None):
++ """Get storage attributes"""
++ _args = ['storage-get', '--format=json']
++ if storage_id:
++ _args.extend(('-s', storage_id))
++ if attribute:
++ _args.append(attribute)
++ try:
++ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
++ except ValueError:
++ return None
++
++
++@cached
++def storage_list(storage_name=None):
++ """List the storage IDs for the unit"""
++ _args = ['storage-list', '--format=json']
++ if storage_name:
++ _args.append(storage_name)
++ try:
++ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
++ except ValueError:
++ return None
++ except OSError as e:
++ import errno
++ if e.errno == errno.ENOENT:
++ # storage-list does not exist
++ return []
++ raise
++
++
  class UnregisteredHookError(Exception):
  """Raised when an undefined hook is called"""
  pass
@@ -643,6 +766,21 @@
  subprocess.check_call(['action-fail', message])
++def action_name():
++ """Get the name of the currently executing action."""
++ return os.environ.get('JUJU_ACTION_NAME')
++
++
++def action_uuid():
++ """Get the UUID of the currently executing action."""
++ return os.environ.get('JUJU_ACTION_UUID')
++
++
++def action_tag():
++ """Get the tag for the currently executing action."""
++ return os.environ.get('JUJU_ACTION_TAG')
++
++
  def status_set(workload_state, message):
  """Set the workload state with a message
@@ -672,25 +810,28 @@
  def status_get():
-- """Retrieve the previously set juju workload state
--
-- If the status-set command is not found then assume this is juju < 1.23 and
-- return 'unknown'
++ """Retrieve the previously set juju workload state and message
++
++ If the status-get command is not found then assume this is juju < 1.23 and
++ return 'unknown', ""
++
  """
-- cmd = ['status-get']
++ cmd = ['status-get', "--format=json", "--include-data"]
  try:
-- raw_status = subprocess.check_output(cmd, universal_newlines=True)
-- status = raw_status.rstrip()
-- return status
++ raw_status = subprocess.check_output(cmd)
  except OSError as e:
  if e.errno == errno.ENOENT:
-- return 'unknown'
++ return ('unknown', "")
  else:
  raise
++ else:
++ status = json.loads(raw_status.decode("UTF-8"))
++ return (status["status"], status["message"])
  def translate_exc(from_exc, to_exc):
  def inner_translate_exc1(f):
++ @wraps(f)
  def inner_translate_exc2(*args, **kwargs):
  try:
  return f(*args, **kwargs)
@@ -702,6 +843,20 @@
  return inner_translate_exc1
++def application_version_set(version):
++ """Charm authors may trigger this command from any hook to output what
++ version of the application is running. This could be a package version,
++ for instance postgres version 9.5. It could also be a build number or
++ version control revision identifier, for instance git sha 6fb7ba68. """
++
++ cmd = ['application-version-set']
++ cmd.append(version)
++ try:
++ subprocess.check_call(cmd)
++ except OSError:
++ log("Application Version: {}".format(version))
++
++
  @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
  def is_leader():
  """Does the current unit hold the juju leadership
@@ -735,6 +890,58 @@
  subprocess.check_call(cmd)
++@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
++def payload_register(ptype, klass, pid):
++ """ is used while a hook is running to let Juju know that a
++ payload has been started."""
++ cmd = ['payload-register']
++ for x in [ptype, klass, pid]:
++ cmd.append(x)
++ subprocess.check_call(cmd)
++
++
++@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
++def payload_unregister(klass, pid):
++ """ is used while a hook is running to let Juju know
++ that a payload has been manually stopped. The <class> and <id> provided
++ must match a payload that has been previously registered with juju using
++ payload-register."""
++ cmd = ['payload-unregister']
++ for x in [klass, pid]:
++ cmd.append(x)
++ subprocess.check_call(cmd)
++
++
++@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
++def payload_status_set(klass, pid, status):
++ """is used to update the current status of a registered payload.
++ The <class> and <id> provided must match a payload that has been previously
++ registered with juju using payload-register. The <status> must be one of the
++ follow: starting, started, stopping, stopped"""
++ cmd = ['payload-status-set']
++ for x in [klass, pid, status]:
++ cmd.append(x)
++ subprocess.check_call(cmd)
++
++
++@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
++def resource_get(name):
++ """used to fetch the resource path of the given name.
++
++ <name> must match a name of defined resource in metadata.yaml
++
++ returns either a path or False if resource not available
++ """
++ if not name:
++ return False
++
++ cmd = ['resource-get', name]
++ try:
++ return subprocess.check_output(cmd).decode('UTF-8')
++ except subprocess.CalledProcessError:
++ return False
++
++
  @cached
  def juju_version():
  """Full version string (eg. '1.23.3.1-trusty-amd64')"""
@@ -799,3 +1006,16 @@
  for callback, args, kwargs in reversed(_atexit):
  callback(*args, **kwargs)
  del _atexit[:]
++
++
++@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
++def network_get_primary_address(binding):
++ '''
++ Retrieve the primary network address for a named binding
++
++ :param binding: string. The name of a relation of extra-binding
++ :return: string. The primary IP address for the named binding
++ :raise: NotImplementedError if run on Juju < 2.0
++ '''
++ cmd = ['network-get', '--primary-address', binding]
++ return subprocess.check_output(cmd).decode('UTF-8').strip()
 === modified file 'hooks/charmhelpers/core/host.py'
 --- hooks/charmhelpers/core/host.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/host.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  """Tools for working with the host system"""
  # Copyright 2012 Canonical Ltd.
@@ -30,13 +28,31 @@
  import string
  import subprocess
  import hashlib
++import functools
++import itertools
++import six
++
  from contextlib import contextmanager
  from collections import OrderedDict
--
--import six
--
  from .hookenv import log
  from .fstab import Fstab
++from charmhelpers.osplatform import get_platform
++
++__platform__ = get_platform()
++if __platform__ == "ubuntu":
++ from charmhelpers.core.host_factory.ubuntu import (
++ service_available,
++ add_new_group,
++ lsb_release,
++ cmp_pkgrevno,
++ ) # flake8: noqa -- ignore F401 for this import
++elif __platform__ == "centos":
++ from charmhelpers.core.host_factory.centos import (
++ service_available,
++ add_new_group,
++ lsb_release,
++ cmp_pkgrevno,
++ ) # flake8: noqa -- ignore F401 for this import
  def service_start(service_name):
@@ -63,47 +79,138 @@
  return service_result
++def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
++ """Pause a system service.
++
++ Stop it, and prevent it from starting again at boot."""
++ stopped = True
++ if service_running(service_name):
++ stopped = service_stop(service_name)
++ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
++ sysv_file = os.path.join(initd_dir, service_name)
++ if init_is_systemd():
++ service('disable', service_name)
++ elif os.path.exists(upstart_file):
++ override_path = os.path.join(
++ init_dir, '{}.override'.format(service_name))
++ with open(override_path, 'w') as fh:
++ fh.write("manual\n")
++ elif os.path.exists(sysv_file):
++ subprocess.check_call(["update-rc.d", service_name, "disable"])
++ else:
++ raise ValueError(
++ "Unable to detect {0} as SystemD, Upstart {1} or"
++ " SysV {2}".format(
++ service_name, upstart_file, sysv_file))
++ return stopped
++
++
++def service_resume(service_name, init_dir="/etc/init",
++ initd_dir="/etc/init.d"):
++ """Resume a system service.
++
++ Reenable starting again at boot. Start the service"""
++ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
++ sysv_file = os.path.join(initd_dir, service_name)
++ if init_is_systemd():
++ service('enable', service_name)
++ elif os.path.exists(upstart_file):
++ override_path = os.path.join(
++ init_dir, '{}.override'.format(service_name))
++ if os.path.exists(override_path):
++ os.unlink(override_path)
++ elif os.path.exists(sysv_file):
++ subprocess.check_call(["update-rc.d", service_name, "enable"])
++ else:
++ raise ValueError(
++ "Unable to detect {0} as SystemD, Upstart {1} or"
++ " SysV {2}".format(
++ service_name, upstart_file, sysv_file))
++
++ started = service_running(service_name)
++ if not started:
++ started = service_start(service_name)
++ return started
++
++
  def service(action, service_name):
  """Control a system service"""
-- cmd = ['service', service_name, action]
++ if init_is_systemd():
++ cmd = ['systemctl', action, service_name]
++ else:
++ cmd = ['service', service_name, action]
  return subprocess.call(cmd) == 0
--def service_running(service):
++_UPSTART_CONF = "/etc/init/{}.conf"
++_INIT_D_CONF = "/etc/init.d/{}"
++
++
++def service_running(service_name):
  """Determine whether a system service is running"""
-- try:
-- output = subprocess.check_output(
-- ['service', service, 'status'],
-- stderr=subprocess.STDOUT).decode('UTF-8')
-- except subprocess.CalledProcessError:
++ if init_is_systemd():
++ return service('is-active', service_name)
++ else:
++ if os.path.exists(_UPSTART_CONF.format(service_name)):
++ try:
++ output = subprocess.check_output(
++ ['status', service_name],
++ stderr=subprocess.STDOUT).decode('UTF-8')
++ except subprocess.CalledProcessError:
++ return False
++ else:
++ # This works for upstart scripts where the 'service' command
++ # returns a consistent string to represent running
++ # 'start/running'
++ if ("start/running" in output or
++ "is running" in output or
++ "up and running" in output):
++ return True
++ elif os.path.exists(_INIT_D_CONF.format(service_name)):
++ # Check System V scripts init script return codes
++ return service('status', service_name)
  return False
-- else:
-- if ("start/running" in output or "is running" in output):
-- return True
-- else:
-- return False
--
--
--def service_available(service_name):
-- """Determine whether a system service is available"""
-- try:
-- subprocess.check_output(
-- ['service', service_name, 'status'],
-- stderr=subprocess.STDOUT).decode('UTF-8')
-- except subprocess.CalledProcessError as e:
-- return b'unrecognized service' not in e.output
-- else:
-- return True
--
--
--def adduser(username, password=None, shell='/bin/bash', system_user=False):
-- """Add a user to the system"""
++
++
++SYSTEMD_SYSTEM = '/run/systemd/system'
++
++
++def init_is_systemd():
++ """Return True if the host system uses systemd, False otherwise."""
++ return os.path.isdir(SYSTEMD_SYSTEM)
++
++
++def adduser(username, password=None, shell='/bin/bash',
++ system_user=False, primary_group=None,
++ secondary_groups=None, uid=None, home_dir=None):
++ """Add a user to the system.
++
++ Will log but otherwise succeed if the user already exists.
++
++ :param str username: Username to create
++ :param str password: Password for user; if ``None``, create a system user
++ :param str shell: The default shell for the user
++ :param bool system_user: Whether to create a login or system user
++ :param str primary_group: Primary group for user; defaults to username
++ :param list secondary_groups: Optional list of additional groups
++ :param int uid: UID for user being created
++ :param str home_dir: Home directory for user
++
++ :returns: The password database entry struct, as returned by `pwd.getpwnam`
++ """
  try:
  user_info = pwd.getpwnam(username)
  log('user {0} already exists!'.format(username))
++ if uid:
++ user_info = pwd.getpwuid(int(uid))
++ log('user with uid {0} already exists!'.format(uid))
  except KeyError:
  log('creating user {0}'.format(username))
  cmd = ['useradd']
++ if uid:
++ cmd.extend(['--uid', str(uid)])
++ if home_dir:
++ cmd.extend(['--home', str(home_dir)])
  if system_user or password is None:
  cmd.append('--system')
  else:
@@ -112,39 +219,89 @@
  '--shell', shell,
  '--password', password,
  ])
++ if not primary_group:
++ try:
++ grp.getgrnam(username)
++ primary_group = username # avoid "group exists" error
++ except KeyError:
++ pass
++ if primary_group:
++ cmd.extend(['-g', primary_group])
++ if secondary_groups:
++ cmd.extend(['-G', ','.join(secondary_groups)])
  cmd.append(username)
  subprocess.check_call(cmd)
  user_info = pwd.getpwnam(username)
  return user_info
--def add_group(group_name, system_group=False):
-- """Add a group to the system"""
++def user_exists(username):
++ """Check if a user exists"""
++ try:
++ pwd.getpwnam(username)
++ user_exists = True
++ except KeyError:
++ user_exists = False
++ return user_exists
++
++
++def uid_exists(uid):
++ """Check if a uid exists"""
++ try:
++ pwd.getpwuid(uid)
++ uid_exists = True
++ except KeyError:
++ uid_exists = False
++ return uid_exists
++
++
++def group_exists(groupname):
++ """Check if a group exists"""
++ try:
++ grp.getgrnam(groupname)
++ group_exists = True
++ except KeyError:
++ group_exists = False
++ return group_exists
++
++
++def gid_exists(gid):
++ """Check if a gid exists"""
++ try:
++ grp.getgrgid(gid)
++ gid_exists = True
++ except KeyError:
++ gid_exists = False
++ return gid_exists
++
++
++def add_group(group_name, system_group=False, gid=None):
++ """Add a group to the system
++
++ Will log but otherwise succeed if the group already exists.
++
++ :param str group_name: group to create
++ :param bool system_group: Create system group
++ :param int gid: GID for user being created
++
++ :returns: The password database entry struct, as returned by `grp.getgrnam`
++ """
  try:
  group_info = grp.getgrnam(group_name)
  log('group {0} already exists!'.format(group_name))
++ if gid:
++ group_info = grp.getgrgid(gid)
++ log('group with gid {0} already exists!'.format(gid))
  except KeyError:
  log('creating group {0}'.format(group_name))
-- cmd = ['addgroup']
-- if system_group:
-- cmd.append('--system')
-- else:
-- cmd.extend([
-- '--group',
-- ])
-- cmd.append(group_name)
-- subprocess.check_call(cmd)
++ add_new_group(group_name, system_group, gid)
  group_info = grp.getgrnam(group_name)
  return group_info
  def add_user_to_group(username, group):
  """Add a user to a group"""
-- cmd = [
-- 'gpasswd', '-a',
-- username,
-- group
-- ]
++ cmd = ['gpasswd', '-a', username, group]
  log("Adding user {} to group {}".format(username, group))
  subprocess.check_call(cmd)
@@ -203,14 +360,12 @@
  def fstab_remove(mp):
-- """Remove the given mountpoint entry from /etc/fstab
-- """
++ """Remove the given mountpoint entry from /etc/fstab"""
  return Fstab.remove_by_mountpoint(mp)
  def fstab_add(dev, mp, fs, options=None):
-- """Adds the given device entry to the /etc/fstab file
-- """
++ """Adds the given device entry to the /etc/fstab file"""
  return Fstab.add(dev, mp, fs, options=options)
@@ -254,9 +409,19 @@
  return system_mounts
++def fstab_mount(mountpoint):
++ """Mount filesystem using fstab"""
++ cmd_args = ['mount', mountpoint]
++ try:
++ subprocess.check_output(cmd_args)
++ except subprocess.CalledProcessError as e:
++ log('Error unmounting {}\n{}'.format(mountpoint, e.output))
++ return False
++ return True
++
++
  def file_hash(path, hash_type='md5'):
-- """
-- Generate a hash checksum of the contents of 'path' or None if not found.
++ """Generate a hash checksum of the contents of 'path' or None if not found.
  :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
  such as md5, sha1, sha256, sha512, etc.
@@ -271,10 +436,9 @@
  def path_hash(path):
-- """
-- Generate a hash checksum of all files matching 'path'. Standard wildcards
-- like '*' and '?' are supported, see documentation for the 'glob' module for
-- more information.
++ """Generate a hash checksum of all files matching 'path'. Standard
++ wildcards like '*' and '?' are supported, see documentation for the 'glob'
++ module for more information.
  :return: dict: A { filename: hash } dictionary for all matched files.
  Empty if none found.
@@ -286,8 +450,7 @@
  def check_hash(path, checksum, hash_type='md5'):
-- """
-- Validate a file using a cryptographic checksum.
++ """Validate a file using a cryptographic checksum.
  :param str checksum: Value of the checksum used to validate the file.
  :param str hash_type: Hash algorithm used to generate `checksum`.
@@ -302,10 +465,11 @@
  class ChecksumError(ValueError):
++ """A class derived from Value error to indicate the checksum failed."""
  pass
--def restart_on_change(restart_map, stopstart=False):
++def restart_on_change(restart_map, stopstart=False, restart_functions=None):
  """Restart services based on configuration files changing
  This function is used a decorator, for example::
@@ -323,35 +487,56 @@
  restarted if any file matching the pattern got changed, created
  or removed. Standard wildcards are supported, see documentation
  for the 'glob' module for more information.
++
++ @param restart_map: {path_file_name: [service_name, ...]
++ @param stopstart: DEFAULT false; whether to stop, start OR restart
++ @param restart_functions: nonstandard functions to use to restart services
++ {svc: func, ...}
++ @returns result from decorated function
  """
  def wrap(f):
++ @functools.wraps(f)
  def wrapped_f(*args, **kwargs):
-- checksums = {path: path_hash(path) for path in restart_map}
-- f(*args, **kwargs)
-- restarts = []
-- for path in restart_map:
-- if path_hash(path) != checksums[path]:
-- restarts += restart_map[path]
-- services_list = list(OrderedDict.fromkeys(restarts))
-- if not stopstart:
-- for service_name in services_list:
-- service('restart', service_name)
-- else:
-- for action in ['stop', 'start']:
-- for service_name in services_list:
-- service(action, service_name)
++ return restart_on_change_helper(
++ (lambda: f(*args, **kwargs)), restart_map, stopstart,
++ restart_functions)
  return wrapped_f
  return wrap
--def lsb_release():
-- """Return /etc/lsb-release in a dict"""
-- d = {}
-- with open('/etc/lsb-release', 'r') as lsb:
-- for l in lsb:
-- k, v = l.split('=')
-- d[k.strip()] = v.strip()
-- return d
++def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
++ restart_functions=None):
++ """Helper function to perform the restart_on_change function.
++
++ This is provided for decorators to restart services if files described
++ in the restart_map have changed after an invocation of lambda_f().
++
++ @param lambda_f: function to call.
++ @param restart_map: {file: [service, ...]}
++ @param stopstart: whether to stop, start or restart a service
++ @param restart_functions: nonstandard functions to use to restart services
++ {svc: func, ...}
++ @returns result of lambda_f()
++ """
++ if restart_functions is None:
++ restart_functions = {}
++ checksums = {path: path_hash(path) for path in restart_map}
++ r = lambda_f()
++ # create a list of lists of the services to restart
++ restarts = [restart_map[path]
++ for path in restart_map
++ if path_hash(path) != checksums[path]]
++ # create a flat list of ordered services without duplicates from lists
++ services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
++ if services_list:
++ actions = ('stop', 'start') if stopstart else ('restart',)
++ for service_name in services_list:
++ if service_name in restart_functions:
++ restart_functions[service_name](service_name)
++ else:
++ for action in actions:
++ service(action, service_name)
++ return r
  def pwgen(length=None):
@@ -370,36 +555,92 @@
  return(''.join(random_chars))
--def list_nics(nic_type):
-- '''Return a list of nics of given type(s)'''
++def is_phy_iface(interface):
++ """Returns True if interface is not virtual, otherwise False."""
++ if interface:
++ sys_net = '/sys/class/net'
++ if os.path.isdir(sys_net):
++ for iface in glob.glob(os.path.join(sys_net, '*')):
++ if '/virtual/' in os.path.realpath(iface):
++ continue
++
++ if interface == os.path.basename(iface):
++ return True
++
++ return False
++
++
++def get_bond_master(interface):
++ """Returns bond master if interface is bond slave otherwise None.
++
++ NOTE: the provided interface is expected to be physical
++ """
++ if interface:
++ iface_path = '/sys/class/net/%s' % (interface)
++ if os.path.exists(iface_path):
++ if '/virtual/' in os.path.realpath(iface_path):
++ return None
++
++ master = os.path.join(iface_path, 'master')
++ if os.path.exists(master):
++ master = os.path.realpath(master)
++ # make sure it is a bond master
++ if os.path.exists(os.path.join(master, 'bonding')):
++ return os.path.basename(master)
++
++ return None
++
++
++def list_nics(nic_type=None):
++ """Return a list of nics of given type(s)"""
  if isinstance(nic_type, six.string_types):
  int_types = [nic_type]
  else:
  int_types = nic_type
++
  interfaces = []
-- for int_type in int_types:
-- cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
++ if nic_type:
++ for int_type in int_types:
++ cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
++ ip_output = subprocess.check_output(cmd).decode('UTF-8')
++ ip_output = ip_output.split('\n')
++ ip_output = (line for line in ip_output if line)
++ for line in ip_output:
++ if line.split()[1].startswith(int_type):
++ matched = re.search('.*: (' + int_type +
++ r'[0-9]+\.[0-9]+)@.*', line)
++ if matched:
++ iface = matched.groups()[0]
++ else:
++ iface = line.split()[1].replace(":", "")
++
++ if iface not in interfaces:
++ interfaces.append(iface)
++ else:
++ cmd = ['ip', 'a']
  ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
-- ip_output = (line for line in ip_output if line)
++ ip_output = (line.strip() for line in ip_output if line)
++
++ key = re.compile('^[0-9]+:\s+(.+):')
  for line in ip_output:
-- if line.split()[1].startswith(int_type):
-- matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
-- if matched:
-- interface = matched.groups()[0]
-- else:
-- interface = line.split()[1].replace(":", "")
-- interfaces.append(interface)
++ matched = re.search(key, line)
++ if matched:
++ iface = matched.group(1)
++ iface = iface.partition("@")[0]
++ if iface not in interfaces:
++ interfaces.append(iface)
  return interfaces
  def set_nic_mtu(nic, mtu):
-- '''Set MTU on a network interface'''
++ """Set the Maximum Transmission Unit (MTU) on a network interface."""
  cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
  subprocess.check_call(cmd)
  def get_nic_mtu(nic):
++ """Return the Maximum Transmission Unit (MTU) for a network interface."""
  cmd = ['ip', 'addr', 'show', nic]
  ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
  mtu = ""
@@ -411,6 +652,7 @@
  def get_nic_hwaddr(nic):
++ """Return the Media Access Control (MAC) for a network interface."""
  cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
  ip_output = subprocess.check_output(cmd).decode('UTF-8')
  hwaddr = ""
@@ -420,35 +662,31 @@
  return hwaddr
--def cmp_pkgrevno(package, revno, pkgcache=None):
-- '''Compare supplied revno with the revno of the installed package
--
-- * 1 => Installed revno is greater than supplied arg
-- * 0 => Installed revno is the same as supplied arg
-- * -1 => Installed revno is less than supplied arg
--
-- This function imports apt_cache function from charmhelpers.fetch if
-- the pkgcache argument is None. Be sure to add charmhelpers.fetch if
-- you call this function, or pass an apt_pkg.Cache() instance.
-- '''
-- import apt_pkg
-- if not pkgcache:
-- from charmhelpers.fetch import apt_cache
-- pkgcache = apt_cache()
-- pkg = pkgcache[package]
-- return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
--
--
  @contextmanager
--def chdir(d):
++def chdir(directory):
++ """Change the current working directory to a different directory for a code
++ block and return the previous directory after the block exits. Useful to
++ run commands from a specificed directory.
++
++ :param str directory: The directory path to change to for this context.
++ """
  cur = os.getcwd()
  try:
-- yield os.chdir(d)
++ yield os.chdir(directory)
  finally:
  os.chdir(cur)
--def chownr(path, owner, group, follow_links=True):
++def chownr(path, owner, group, follow_links=True, chowntopdir=False):
++ """Recursively change user and group ownership of files and directories
++ in given path. Doesn't chown path itself by default, only its children.
++
++ :param str path: The string path to start changing ownership.
++ :param str owner: The owner string to use when looking up the uid.
++ :param str group: The group string to use when looking up the gid.
++ :param bool follow_links: Also Chown links if True
++ :param bool chowntopdir: Also chown path itself if True
++ """
  uid = pwd.getpwnam(owner).pw_uid
  gid = grp.getgrnam(group).gr_gid
  if follow_links:
@@ -456,6 +694,10 @@
  else:
  chown = os.lchown
++ if chowntopdir:
++ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
++ if not broken_symlink:
++ chown(path, uid, gid)
  for root, dirs, files in os.walk(path):
  for name in dirs + files:
  full = os.path.join(root, name)
@@ -465,4 +707,28 @@
  def lchownr(path, owner, group):
++ """Recursively change user and group ownership of files and directories
++ in a given path, not following symbolic links. See the documentation for
++ 'os.lchown' for more information.
++
++ :param str path: The string path to start changing ownership.
++ :param str owner: The owner string to use when looking up the uid.
++ :param str group: The group string to use when looking up the gid.
++ """
  chownr(path, owner, group, follow_links=False)
++
++
++def get_total_ram():
++ """The total amount of system RAM in bytes.
++
++ This is what is reported by the OS, and may be overcommitted when
++ there are multiple containers hosted on the same machine.
++ """
++ with open('/proc/meminfo', 'r') as f:
++ for line in f.readlines():
++ if line:
++ key, value, unit = line.split()
++ if key == 'MemTotal:':
++ assert unit == 'kB', 'Unknown unit'
++ return int(value) * 1024 # Classic, not KiB.
++ raise NotImplementedError()
 === added directory 'hooks/charmhelpers/core/host_factory'
 === added file 'hooks/charmhelpers/core/host_factory/__init__.py'
 === added file 'hooks/charmhelpers/core/host_factory/centos.py'
 --- hooks/charmhelpers/core/host_factory/centos.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/host_factory/centos.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,56 @@
++import subprocess
++import yum
++import os
++
++
++def service_available(service_name):
++ # """Determine whether a system service is available."""
++ if os.path.isdir('/run/systemd/system'):
++ cmd = ['systemctl', 'is-enabled', service_name]
++ else:
++ cmd = ['service', service_name, 'is-enabled']
++ return subprocess.call(cmd) == 0
++
++
++def add_new_group(group_name, system_group=False, gid=None):
++ cmd = ['groupadd']
++ if gid:
++ cmd.extend(['--gid', str(gid)])
++ if system_group:
++ cmd.append('-r')
++ cmd.append(group_name)
++ subprocess.check_call(cmd)
++
++
++def lsb_release():
++ """Return /etc/os-release in a dict."""
++ d = {}
++ with open('/etc/os-release', 'r') as lsb:
++ for l in lsb:
++ s = l.split('=')
++ if len(s) != 2:
++ continue
++ d[s[0].strip()] = s[1].strip()
++ return d
++
++
++def cmp_pkgrevno(package, revno, pkgcache=None):
++ """Compare supplied revno with the revno of the installed package.
++
++ * 1 => Installed revno is greater than supplied arg
++ * 0 => Installed revno is the same as supplied arg
++ * -1 => Installed revno is less than supplied arg
++
++ This function imports YumBase function if the pkgcache argument
++ is None.
++ """
++ if not pkgcache:
++ y = yum.YumBase()
++ packages = y.doPackageLists()
++ pkgcache = {i.Name: i.version for i in packages['installed']}
++ pkg = pkgcache[package]
++ if pkg > revno:
++ return 1
++ if pkg < revno:
++ return -1
++ return 0
 === added file 'hooks/charmhelpers/core/host_factory/ubuntu.py'
 --- hooks/charmhelpers/core/host_factory/ubuntu.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/host_factory/ubuntu.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,56 @@
++import subprocess
++
++
++def service_available(service_name):
++ """Determine whether a system service is available"""
++ try:
++ subprocess.check_output(
++ ['service', service_name, 'status'],
++ stderr=subprocess.STDOUT).decode('UTF-8')
++ except subprocess.CalledProcessError as e:
++ return b'unrecognized service' not in e.output
++ else:
++ return True
++
++
++def add_new_group(group_name, system_group=False, gid=None):
++ cmd = ['addgroup']
++ if gid:
++ cmd.extend(['--gid', str(gid)])
++ if system_group:
++ cmd.append('--system')
++ else:
++ cmd.extend([
++ '--group',
++ ])
++ cmd.append(group_name)
++ subprocess.check_call(cmd)
++
++
++def lsb_release():
++ """Return /etc/lsb-release in a dict"""
++ d = {}
++ with open('/etc/lsb-release', 'r') as lsb:
++ for l in lsb:
++ k, v = l.split('=')
++ d[k.strip()] = v.strip()
++ return d
++
++
++def cmp_pkgrevno(package, revno, pkgcache=None):
++ """Compare supplied revno with the revno of the installed package.
++
++ * 1 => Installed revno is greater than supplied arg
++ * 0 => Installed revno is the same as supplied arg
++ * -1 => Installed revno is less than supplied arg
++
++ This function imports apt_cache function from charmhelpers.fetch if
++ the pkgcache argument is None. Be sure to add charmhelpers.fetch if
++ you call this function, or pass an apt_pkg.Cache() instance.
++ """
++ import apt_pkg
++ if not pkgcache:
++ from charmhelpers.fetch import apt_cache
++ pkgcache = apt_cache()
++ pkg = pkgcache[package]
++ return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
 === added file 'hooks/charmhelpers/core/hugepage.py'
 --- hooks/charmhelpers/core/hugepage.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/hugepage.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,69 @@
++# -*- coding: utf-8 -*-
++
++# Copyright 2014-2015 Canonical Limited.
++#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
++
++import yaml
++from charmhelpers.core import fstab
++from charmhelpers.core import sysctl
++from charmhelpers.core.host import (
++ add_group,
++ add_user_to_group,
++ fstab_mount,
++ mkdir,
++)
++from charmhelpers.core.strutils import bytes_from_string
++from subprocess import check_output
++
++
++def hugepage_support(user, group='hugetlb', nr_hugepages=256,
++ max_map_count=65536, mnt_point='/run/hugepages/kvm',
++ pagesize='2MB', mount=True, set_shmmax=False):
++ """Enable hugepages on system.
++
++ Args:
++ user (str) -- Username to allow access to hugepages to
++ group (str) -- Group name to own hugepages
++ nr_hugepages (int) -- Number of pages to reserve
++ max_map_count (int) -- Number of Virtual Memory Areas a process can own
++ mnt_point (str) -- Directory to mount hugepages on
++ pagesize (str) -- Size of hugepages
++ mount (bool) -- Whether to Mount hugepages
++ """
++ group_info = add_group(group)
++ gid = group_info.gr_gid
++ add_user_to_group(user, group)
++ if max_map_count < 2 * nr_hugepages:
++ max_map_count = 2 * nr_hugepages
++ sysctl_settings = {
++ 'vm.nr_hugepages': nr_hugepages,
++ 'vm.max_map_count': max_map_count,
++ 'vm.hugetlb_shm_group': gid,
++ }
++ if set_shmmax:
++ shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
++ shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
++ if shmmax_minsize > shmmax_current:
++ sysctl_settings['kernel.shmmax'] = shmmax_minsize
++ sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
++ mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
++ lfstab = fstab.Fstab()
++ fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
++ if fstab_entry:
++ lfstab.remove_entry(fstab_entry)
++ entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
++ 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
++ lfstab.add_entry(entry)
++ if mount:
++ fstab_mount(mnt_point)
 === added file 'hooks/charmhelpers/core/kernel.py'
 --- hooks/charmhelpers/core/kernel.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/kernel.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,72 @@
++#!/usr/bin/env python
++# -*- coding: utf-8 -*-
++
++# Copyright 2014-2015 Canonical Limited.
++#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
++
++import re
++import subprocess
++
++from charmhelpers.osplatform import get_platform
++from charmhelpers.core.hookenv import (
++ log,
++ INFO
++)
++
++__platform__ = get_platform()
++if __platform__ == "ubuntu":
++ from charmhelpers.core.kernel_factory.ubuntu import (
++ persistent_modprobe,
++ update_initramfs,
++ ) # flake8: noqa -- ignore F401 for this import
++elif __platform__ == "centos":
++ from charmhelpers.core.kernel_factory.centos import (
++ persistent_modprobe,
++ update_initramfs,
++ ) # flake8: noqa -- ignore F401 for this import
++
++__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
++
++
++def modprobe(module, persist=True):
++ """Load a kernel module and configure for auto-load on reboot."""
++ cmd = ['modprobe', module]
++
++ log('Loading kernel module %s' % module, level=INFO)
++
++ subprocess.check_call(cmd)
++ if persist:
++ persistent_modprobe(module)
++
++
++def rmmod(module, force=False):
++ """Remove a module from the linux kernel"""
++ cmd = ['rmmod']
++ if force:
++ cmd.append('-f')
++ cmd.append(module)
++ log('Removing kernel module %s' % module, level=INFO)
++ return subprocess.check_call(cmd)
++
++
++def lsmod():
++ """Shows what kernel modules are currently loaded"""
++ return subprocess.check_output(['lsmod'],
++ universal_newlines=True)
++
++
++def is_module_loaded(module):
++ """Checks if a kernel module is already loaded"""
++ matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
++ return len(matches) > 0
 === added directory 'hooks/charmhelpers/core/kernel_factory'
 === added file 'hooks/charmhelpers/core/kernel_factory/__init__.py'
 === added file 'hooks/charmhelpers/core/kernel_factory/centos.py'
 --- hooks/charmhelpers/core/kernel_factory/centos.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/kernel_factory/centos.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,17 @@
++import subprocess
++import os
++
++
++def persistent_modprobe(module):
++ """Load a kernel module and configure for auto-load on reboot."""
++ if not os.path.exists('/etc/rc.modules'):
++ open('/etc/rc.modules', 'a')
++ os.chmod('/etc/rc.modules', 111)
++ with open('/etc/rc.modules', 'r+') as modules:
++ if module not in modules.read():
++ modules.write('modprobe %s\n' % module)
++
++
++def update_initramfs(version='all'):
++ """Updates an initramfs image."""
++ return subprocess.check_call(["dracut", "-f", version])
 === added file 'hooks/charmhelpers/core/kernel_factory/ubuntu.py'
 --- hooks/charmhelpers/core/kernel_factory/ubuntu.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/core/kernel_factory/ubuntu.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,13 @@
++import subprocess
++
++
++def persistent_modprobe(module):
++ """Load a kernel module and configure for auto-load on reboot."""
++ with open('/etc/modules', 'r+') as modules:
++ if module not in modules.read():
++ modules.write(module)
++
++
++def update_initramfs(version='all'):
++ """Updates an initramfs image."""
++ return subprocess.check_call(["update-initramfs", "-k", version, "-u"])
 === modified file 'hooks/charmhelpers/core/services/__init__.py'
 --- hooks/charmhelpers/core/services/__init__.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/core/services/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  from .base import * # NOQA
  from .helpers import * # NOQA
 === modified file 'hooks/charmhelpers/core/services/base.py'
 --- hooks/charmhelpers/core/services/base.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/services/base.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
  import json
 === modified file 'hooks/charmhelpers/core/services/helpers.py'
 --- hooks/charmhelpers/core/services/helpers.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/services/helpers.py	2016-09-16 19:19:06 +0000
@@ -1,22 +1,22 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
  import yaml
++
  from charmhelpers.core import hookenv
++from charmhelpers.core import host
  from charmhelpers.core import templating
  from charmhelpers.core.services.base import ManagerCallback
@@ -240,27 +240,50 @@
  :param str source: The template source file, relative to
  `$CHARM_DIR/templates`
-- :param str target: The target to write the rendered template to
++
++ :param str target: The target to write the rendered template to (or None)
  :param str owner: The owner of the rendered file
  :param str group: The group of the rendered file
  :param int perms: The permissions of the rendered file
++ :param partial on_change_action: functools partial to be executed when
++ rendered file changes
++ :param jinja2 loader template_loader: A jinja2 template loader
++ :return str: The rendered template
  """
  def __init__(self, source, target,
-- owner='root', group='root', perms=0o444):
++ owner='root', group='root', perms=0o444,
++ on_change_action=None, template_loader=None):
  self.source = source
  self.target = target
  self.owner = owner
  self.group = group
  self.perms = perms
++ self.on_change_action = on_change_action
++ self.template_loader = template_loader
  def __call__(self, manager, service_name, event_name):
++ pre_checksum = ''
++ if self.on_change_action and os.path.isfile(self.target):
++ pre_checksum = host.file_hash(self.target)
  service = manager.get_service(service_name)
-- context = {}
++ context = {'ctx': {}}
  for ctx in service.get('required_data', []):
  context.update(ctx)
-- templating.render(self.source, self.target, context,
-- self.owner, self.group, self.perms)
++ context['ctx'].update(ctx)
++
++ result = templating.render(self.source, self.target, context,
++ self.owner, self.group, self.perms,
++ template_loader=self.template_loader)
++ if self.on_change_action:
++ if pre_checksum == host.file_hash(self.target):
++ hookenv.log(
++ 'No change detected: {}'.format(self.target),
++ hookenv.DEBUG)
++ else:
++ self.on_change_action()
++
++ return result
  # Convenience aliases for templates
 === modified file 'hooks/charmhelpers/core/strutils.py'
 --- hooks/charmhelpers/core/strutils.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/strutils.py	2016-09-16 19:19:06 +0000
@@ -3,21 +3,20 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import six
++import re
  def bool_from_string(value):
@@ -40,3 +39,32 @@
  msg = "Unable to interpret string value '%s' as boolean" % (value)
  raise ValueError(msg)
++
++
++def bytes_from_string(value):
++ """Interpret human readable string value as bytes.
++
++ Returns int
++ """
++ BYTE_POWER = {
++ 'K': 1,
++ 'KB': 1,
++ 'M': 2,
++ 'MB': 2,
++ 'G': 3,
++ 'GB': 3,
++ 'T': 4,
++ 'TB': 4,
++ 'P': 5,
++ 'PB': 5,
++ }
++ if isinstance(value, six.string_types):
++ value = six.text_type(value)
++ else:
++ msg = "Unable to interpret non-string value '%s' as boolean" % (value)
++ raise ValueError(msg)
++ matches = re.match("([0-9]+)([a-zA-Z]+)", value)
++ if not matches:
++ msg = "Unable to interpret string value '%s' as bytes" % (value)
++ raise ValueError(msg)
++ return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
 === modified file 'hooks/charmhelpers/core/sysctl.py'
 --- hooks/charmhelpers/core/sysctl.py	2015-02-12 20:10:50 +0000
 +++ hooks/charmhelpers/core/sysctl.py	2016-09-16 19:19:06 +0000
@@ -3,19 +3,17 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import yaml
 === modified file 'hooks/charmhelpers/core/templating.py'
 --- hooks/charmhelpers/core/templating.py	2015-02-04 12:16:14 +0000
 +++ hooks/charmhelpers/core/templating.py	2016-09-16 19:19:06 +0000
@@ -1,33 +1,33 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
++import sys
  from charmhelpers.core import host
  from charmhelpers.core import hookenv
  def render(source, target, context, owner='root', group='root',
-- perms=0o444, templates_dir=None, encoding='UTF-8'):
++ perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
  """
  Render a template.
  The `source` path, if not absolute, is relative to the `templates_dir`.
-- The `target` path should be absolute.
++ The `target` path should be absolute. It can also be `None`, in which
++ case no file will be written.
  The context should be a dict containing the values to be replaced in the
  template.
@@ -36,8 +36,12 @@
  If omitted, `templates_dir` defaults to the `templates` folder in the charm.
-- Note: Using this requires python-jinja2; if it is not installed, calling
-- this will attempt to use charmhelpers.fetch.apt_install to install it.
++ The rendered template will be written to the file as well as being returned
++ as a string.
++
++ Note: Using this requires python-jinja2 or python3-jinja2; if it is not
++ installed, calling this will attempt to use charmhelpers.fetch.apt_install
++ to install it.
  """
  try:
  from jinja2 import FileSystemLoader, Environment, exceptions
@@ -49,20 +53,32 @@
  'charmhelpers.fetch to install it',
  level=hookenv.ERROR)
  raise
-- apt_install('python-jinja2', fatal=True)
++ if sys.version_info.major == 2:
++ apt_install('python-jinja2', fatal=True)
++ else:
++ apt_install('python3-jinja2', fatal=True)
  from jinja2 import FileSystemLoader, Environment, exceptions
-- if templates_dir is None:
-- templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
-- loader = Environment(loader=FileSystemLoader(templates_dir))
++ if template_loader:
++ template_env = Environment(loader=template_loader)
++ else:
++ if templates_dir is None:
++ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
++ template_env = Environment(loader=FileSystemLoader(templates_dir))
  try:
  source = source
-- template = loader.get_template(source)
++ template = template_env.get_template(source)
  except exceptions.TemplateNotFound as e:
  hookenv.log('Could not load template %s from %s.' %
  (source, templates_dir),
  level=hookenv.ERROR)
  raise e
  content = template.render(context)
-- host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
-- host.write_file(target, content.encode(encoding), owner, group, perms)
++ if target is not None:
++ target_dir = os.path.dirname(target)
++ if not os.path.exists(target_dir):
++ # This is a terrible default directory permission, as the file
++ # or its siblings will often contain secrets.
++ host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
++ host.write_file(target, content.encode(encoding), owner, group, perms)
++ return content
 === modified file 'hooks/charmhelpers/core/unitdata.py'
 --- hooks/charmhelpers/core/unitdata.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/core/unitdata.py	2016-09-16 19:19:06 +0000
@@ -3,20 +3,17 @@
+ #
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
--#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
+ #
  # Authors:
  # Kapil Thangavelu <kapil.foss@gmail.com>
@@ -152,6 +149,7 @@
  import collections
  import contextlib
  import datetime
++import itertools
  import json
  import os
  import pprint
@@ -164,8 +162,7 @@
  class Storage(object):
  """Simple key value database for local unit state within charms.
-- Modifications are automatically committed at hook exit. That's
-- currently regardless of exit code.
++ Modifications are not persisted unless :meth:`flush` is called.
  To support dicts, lists, integer, floats, and booleans values
  are automatically json encoded/decoded.
@@ -173,8 +170,11 @@
  def __init__(self, path=None):
  self.db_path = path
  if path is None:
-- self.db_path = os.path.join(
-- os.environ.get('CHARM_DIR', ''), '.unit-state.db')
++ if 'UNIT_STATE_DB' in os.environ:
++ self.db_path = os.environ['UNIT_STATE_DB']
++ else:
++ self.db_path = os.path.join(
++ os.environ.get('CHARM_DIR', ''), '.unit-state.db')
  self.conn = sqlite3.connect('%s' % self.db_path)
  self.cursor = self.conn.cursor()
  self.revision = None
@@ -189,15 +189,8 @@
  self.conn.close()
  self._closed = True
-- def _scoped_query(self, stmt, params=None):
-- if params is None:
-- params = []
-- return stmt, params
--
  def get(self, key, default=None, record=False):
-- self.cursor.execute(
-- *self._scoped_query(
-- 'select data from kv where key=?', [key]))
++ self.cursor.execute('select data from kv where key=?', [key])
  result = self.cursor.fetchone()
  if not result:
  return default
@@ -206,33 +199,81 @@
  return json.loads(result[0])
  def getrange(self, key_prefix, strip=False):
-- stmt = "select key, data from kv where key like '%s%%'" % key_prefix
-- self.cursor.execute(*self._scoped_query(stmt))
++ """
++ Get a range of keys starting with a common prefix as a mapping of
++ keys to values.
++
++ :param str key_prefix: Common prefix among all keys
++ :param bool strip: Optionally strip the common prefix from the key
++ names in the returned dict
++ :return dict: A (possibly empty) dict of key-value mappings
++ """
++ self.cursor.execute("select key, data from kv where key like ?",
++ ['%s%%' % key_prefix])
  result = self.cursor.fetchall()
  if not result:
-- return None
++ return {}
  if not strip:
  key_prefix = ''
  return dict([
  (k[len(key_prefix):], json.loads(v)) for k, v in result])
  def update(self, mapping, prefix=""):
++ """
++ Set the values of multiple keys at once.
++
++ :param dict mapping: Mapping of keys to values
++ :param str prefix: Optional prefix to apply to all keys in `mapping`
++ before setting
++ """
  for k, v in mapping.items():
  self.set("%s%s" % (prefix, k), v)
  def unset(self, key):
++ """
++ Remove a key from the database entirely.
++ """
  self.cursor.execute('delete from kv where key=?', [key])
  if self.revision and self.cursor.rowcount:
  self.cursor.execute(
  'insert into kv_revisions values (?, ?, ?)',
  [key, self.revision, json.dumps('DELETED')])
++ def unsetrange(self, keys=None, prefix=""):
++ """
++ Remove a range of keys starting with a common prefix, from the database
++ entirely.
++
++ :param list keys: List of keys to remove.
++ :param str prefix: Optional prefix to apply to all keys in ``keys``
++ before removing.
++ """
++ if keys is not None:
++ keys = ['%s%s' % (prefix, key) for key in keys]
++ self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
++ if self.revision and self.cursor.rowcount:
++ self.cursor.execute(
++ 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
++ list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
++ else:
++ self.cursor.execute('delete from kv where key like ?',
++ ['%s%%' % prefix])
++ if self.revision and self.cursor.rowcount:
++ self.cursor.execute(
++ 'insert into kv_revisions values (?, ?, ?)',
++ ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
++
  def set(self, key, value):
++ """
++ Set a value in the database.
++
++ :param str key: Key to set the value for
++ :param value: Any JSON-serializable value to be set
++ """
  serialized = json.dumps(value)
-- self.cursor.execute(
-- 'select data from kv where key=?', [key])
++ self.cursor.execute('select data from kv where key=?', [key])
  exists = self.cursor.fetchone()
  # Skip mutations to the same value
 === modified file 'hooks/charmhelpers/fetch/__init__.py'
 --- hooks/charmhelpers/fetch/__init__.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/fetch/__init__.py	2016-09-16 19:19:06 +0000
@@ -1,32 +1,24 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import importlib
--from tempfile import NamedTemporaryFile
--import time
++from charmhelpers.osplatform import get_platform
  from yaml import safe_load
--from charmhelpers.core.host import (
-- lsb_release
--)
--import subprocess
  from charmhelpers.core.hookenv import (
  config,
  log,
+ )
--import os
  import six
  if six.PY3:
@@ -35,63 +27,6 @@
  from urlparse import urlparse, urlunparse
--CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
--deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
--"""
--PROPOSED_POCKET = """# Proposed
--deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
--"""
--CLOUD_ARCHIVE_POCKETS = {
-- # Folsom
-- 'folsom': 'precise-updates/folsom',
-- 'precise-folsom': 'precise-updates/folsom',
-- 'precise-folsom/updates': 'precise-updates/folsom',
-- 'precise-updates/folsom': 'precise-updates/folsom',
-- 'folsom/proposed': 'precise-proposed/folsom',
-- 'precise-folsom/proposed': 'precise-proposed/folsom',
-- 'precise-proposed/folsom': 'precise-proposed/folsom',
-- # Grizzly
-- 'grizzly': 'precise-updates/grizzly',
-- 'precise-grizzly': 'precise-updates/grizzly',
-- 'precise-grizzly/updates': 'precise-updates/grizzly',
-- 'precise-updates/grizzly': 'precise-updates/grizzly',
-- 'grizzly/proposed': 'precise-proposed/grizzly',
-- 'precise-grizzly/proposed': 'precise-proposed/grizzly',
-- 'precise-proposed/grizzly': 'precise-proposed/grizzly',
-- # Havana
-- 'havana': 'precise-updates/havana',
-- 'precise-havana': 'precise-updates/havana',
-- 'precise-havana/updates': 'precise-updates/havana',
-- 'precise-updates/havana': 'precise-updates/havana',
-- 'havana/proposed': 'precise-proposed/havana',
-- 'precise-havana/proposed': 'precise-proposed/havana',
-- 'precise-proposed/havana': 'precise-proposed/havana',
-- # Icehouse
-- 'icehouse': 'precise-updates/icehouse',
-- 'precise-icehouse': 'precise-updates/icehouse',
-- 'precise-icehouse/updates': 'precise-updates/icehouse',
-- 'precise-updates/icehouse': 'precise-updates/icehouse',
-- 'icehouse/proposed': 'precise-proposed/icehouse',
-- 'precise-icehouse/proposed': 'precise-proposed/icehouse',
-- 'precise-proposed/icehouse': 'precise-proposed/icehouse',
-- # Juno
-- 'juno': 'trusty-updates/juno',
-- 'trusty-juno': 'trusty-updates/juno',
-- 'trusty-juno/updates': 'trusty-updates/juno',
-- 'trusty-updates/juno': 'trusty-updates/juno',
-- 'juno/proposed': 'trusty-proposed/juno',
-- 'trusty-juno/proposed': 'trusty-proposed/juno',
-- 'trusty-proposed/juno': 'trusty-proposed/juno',
-- # Kilo
-- 'kilo': 'trusty-updates/kilo',
-- 'trusty-kilo': 'trusty-updates/kilo',
-- 'trusty-kilo/updates': 'trusty-updates/kilo',
-- 'trusty-updates/kilo': 'trusty-updates/kilo',
-- 'kilo/proposed': 'trusty-proposed/kilo',
-- 'trusty-kilo/proposed': 'trusty-proposed/kilo',
-- 'trusty-proposed/kilo': 'trusty-proposed/kilo',
--}
--
  # The order of this list is very important. Handlers should be listed in from
  # least- to most-specific URL matching.
  FETCH_HANDLERS = (
@@ -100,10 +35,6 @@
  'charmhelpers.fetch.giturl.GitUrlFetchHandler',
+ )
--APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
--APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
--APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
--
  class SourceConfigError(Exception):
  pass
@@ -141,180 +72,37 @@
  return urlunparse(parts)
--def filter_installed_packages(packages):
-- """Returns a list of packages that require installation"""
-- cache = apt_cache()
-- _pkgs = []
-- for package in packages:
-- try:
-- p = cache[package]
-- p.current_ver or _pkgs.append(package)
-- except KeyError:
-- log('Package {} has no installation candidate.'.format(package),
-- level='WARNING')
-- _pkgs.append(package)
-- return _pkgs
--
--
--def apt_cache(in_memory=True):
-- """Build and return an apt cache"""
-- from apt import apt_pkg
-- apt_pkg.init()
-- if in_memory:
-- apt_pkg.config.set("Dir::Cache::pkgcache", "")
-- apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
-- return apt_pkg.Cache()
--
--
--def apt_install(packages, options=None, fatal=False):
-- """Install one or more packages"""
-- if options is None:
-- options = ['--option=Dpkg::Options::=--force-confold']
--
-- cmd = ['apt-get', '--assume-yes']
-- cmd.extend(options)
-- cmd.append('install')
-- if isinstance(packages, six.string_types):
-- cmd.append(packages)
-- else:
-- cmd.extend(packages)
-- log("Installing {} with options: {}".format(packages,
-- options))
-- _run_apt_command(cmd, fatal)
--
--
--def apt_upgrade(options=None, fatal=False, dist=False):
-- """Upgrade all packages"""
-- if options is None:
-- options = ['--option=Dpkg::Options::=--force-confold']
--
-- cmd = ['apt-get', '--assume-yes']
-- cmd.extend(options)
-- if dist:
-- cmd.append('dist-upgrade')
-- else:
-- cmd.append('upgrade')
-- log("Upgrading with options: {}".format(options))
-- _run_apt_command(cmd, fatal)
--
--
--def apt_update(fatal=False):
-- """Update local apt cache"""
-- cmd = ['apt-get', 'update']
-- _run_apt_command(cmd, fatal)
--
--
--def apt_purge(packages, fatal=False):
-- """Purge one or more packages"""
-- cmd = ['apt-get', '--assume-yes', 'purge']
-- if isinstance(packages, six.string_types):
-- cmd.append(packages)
-- else:
-- cmd.extend(packages)
-- log("Purging {}".format(packages))
-- _run_apt_command(cmd, fatal)
--
--
--def apt_mark(packages, mark, fatal=False):
-- """Flag one or more packages using apt-mark"""
-- cmd = ['apt-mark', mark]
-- if isinstance(packages, six.string_types):
-- cmd.append(packages)
-- else:
-- cmd.extend(packages)
-- log("Holding {}".format(packages))
--
-- if fatal:
-- subprocess.check_call(cmd, universal_newlines=True)
-- else:
-- subprocess.call(cmd, universal_newlines=True)
--
--
--def apt_hold(packages, fatal=False):
-- return apt_mark(packages, 'hold', fatal=fatal)
--
--
--def apt_unhold(packages, fatal=False):
-- return apt_mark(packages, 'unhold', fatal=fatal)
--
--
--def add_source(source, key=None):
-- """Add a package source to this system.
--
-- @param source: a URL or sources.list entry, as supported by
-- add-apt-repository(1). Examples::
--
-- ppa:charmers/example
-- deb https://stub:key@private.example.com/ubuntu trusty main
--
-- In addition:
-- 'proposed:' may be used to enable the standard 'proposed'
-- pocket for the release.
-- 'cloud:' may be used to activate official cloud archive pockets,
-- such as 'cloud:icehouse'
-- 'distro' may be used as a noop
--
-- @param key: A key to be added to the system's APT keyring and used
-- to verify the signatures on packages. Ideally, this should be an
-- ASCII format GPG public key including the block headers. A GPG key
-- id may also be used, but be aware that only insecure protocols are
-- available to retrieve the actual public key from a public keyserver
-- placing your Juju environment at risk. ppa and cloud archive keys
-- are securely added automtically, so sould not be provided.
-- """
-- if source is None:
-- log('Source is not present. Skipping')
-- return
--
-- if (source.startswith('ppa:') or
-- source.startswith('http') or
-- source.startswith('deb ') or
-- source.startswith('cloud-archive:')):
-- subprocess.check_call(['add-apt-repository', '--yes', source])
-- elif source.startswith('cloud:'):
-- apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
-- fatal=True)
-- pocket = source.split(':')[-1]
-- if pocket not in CLOUD_ARCHIVE_POCKETS:
-- raise SourceConfigError(
-- 'Unsupported cloud: source option %s' %
-- pocket)
-- actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
-- with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
-- apt.write(CLOUD_ARCHIVE.format(actual_pocket))
-- elif source == 'proposed':
-- release = lsb_release()['DISTRIB_CODENAME']
-- with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
-- apt.write(PROPOSED_POCKET.format(release))
-- elif source == 'distro':
-- pass
-- else:
-- log("Unknown source: {!r}".format(source))
--
-- if key:
-- if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
-- with NamedTemporaryFile('w+') as key_file:
-- key_file.write(key)
-- key_file.flush()
-- key_file.seek(0)
-- subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
-- else:
-- # Note that hkp: is in no way a secure protocol. Using a
-- # GPG key id is pointless from a security POV unless you
-- # absolutely trust your network and DNS.
-- subprocess.check_call(['apt-key', 'adv', '--keyserver',
-- 'hkp://keyserver.ubuntu.com:80', '--recv',
-- key])
++__platform__ = get_platform()
++module = "charmhelpers.fetch.%s" % __platform__
++fetch = importlib.import_module(module)
++
++filter_installed_packages = fetch.filter_installed_packages
++install = fetch.install
++upgrade = fetch.upgrade
++update = fetch.update
++purge = fetch.purge
++add_source = fetch.add_source
++
++if __platform__ == "ubuntu":
++ apt_cache = fetch.apt_cache
++ apt_install = fetch.install
++ apt_update = fetch.update
++ apt_upgrade = fetch.upgrade
++ apt_purge = fetch.purge
++ apt_mark = fetch.apt_mark
++ apt_hold = fetch.apt_hold
++ apt_unhold = fetch.apt_unhold
++elif __platform__ == "centos":
++ yum_search = fetch.yum_search
  def configure_sources(update=False,
  sources_var='install_sources',
  keys_var='install_keys'):
-- """
-- Configure multiple sources from charm configuration.
++ """Configure multiple sources from charm configuration.
  The lists are encoded as yaml fragments in the configuration.
-- The frament needs to be included as a string. Sources and their
++ The fragment needs to be included as a string. Sources and their
  corresponding keys are of the types supported by add_source().
  Example config:
@@ -346,12 +134,11 @@
  for source, key in zip(sources, keys):
  add_source(source, key)
  if update:
-- apt_update(fatal=True)
++ fetch.update(fatal=True)
  def install_remote(source, *args, **kwargs):
-- """
-- Install a file tree from a remote source
++ """Install a file tree from a remote source.
  The specified source should be a url of the form:
  scheme://[host]/path[#[option=value][&...]]
@@ -374,18 +161,17 @@
  # We ONLY check for True here because can_handle may return a string
  # explaining why it can't handle a given source.
  handlers = [h for h in plugins() if h.can_handle(source) is True]
-- installed_to = None
  for handler in handlers:
  try:
-- installed_to = handler.install(source, *args, **kwargs)
-- except UnhandledSource:
-- pass
-- if not installed_to:
-- raise UnhandledSource("No handler found for source {}".format(source))
-- return installed_to
++ return handler.install(source, *args, **kwargs)
++ except UnhandledSource as e:
++ log('Install source attempt unsuccessful: {}'.format(e),
++ level='WARNING')
++ raise UnhandledSource("No handler found for source {}".format(source))
  def install_from_config(config_var_name):
++ """Install a file from config."""
  charm_config = config()
  source = charm_config[config_var_name]
  return install_remote(source)
@@ -402,46 +188,9 @@
  importlib.import_module(package),
  classname)
  plugin_list.append(handler_class())
-- except (ImportError, AttributeError):
++ except NotImplementedError:
  # Skip missing plugins so that they can be ommitted from
  # installation if desired
  log("FetchHandler {} not found, skipping plugin".format(
  handler_name))
  return plugin_list
--
--
--def _run_apt_command(cmd, fatal=False):
-- """
-- Run an APT command, checking output and retrying if the fatal flag is set
-- to True.
--
-- :param: cmd: str: The apt command to run.
-- :param: fatal: bool: Whether the command's output should be checked and
-- retried.
-- """
-- env = os.environ.copy()
--
-- if 'DEBIAN_FRONTEND' not in env:
-- env['DEBIAN_FRONTEND'] = 'noninteractive'
--
-- if fatal:
-- retry_count = 0
-- result = None
--
-- # If the command is considered "fatal", we need to retry if the apt
-- # lock was not acquired.
--
-- while result is None or result == APT_NO_LOCK:
-- try:
-- result = subprocess.check_call(cmd, env=env)
-- except subprocess.CalledProcessError as e:
-- retry_count = retry_count + 1
-- if retry_count > APT_NO_LOCK_RETRY_COUNT:
-- raise
-- result = e.returncode
-- log("Couldn't acquire DPKG lock. Will retry in {} seconds."
-- "".format(APT_NO_LOCK_RETRY_DELAY))
-- time.sleep(APT_NO_LOCK_RETRY_DELAY)
--
-- else:
-- subprocess.call(cmd, env=env)
 === modified file 'hooks/charmhelpers/fetch/archiveurl.py'
 --- hooks/charmhelpers/fetch/archiveurl.py	2015-02-12 20:10:50 +0000
 +++ hooks/charmhelpers/fetch/archiveurl.py	2016-09-16 19:19:06 +0000
@@ -1,18 +1,16 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
  import hashlib
@@ -77,6 +75,8 @@
  def can_handle(self, source):
  url_parts = self.parse_url(source)
  if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
++ # XXX: Why is this returning a boolean and a string? It's
++ # doomed to fail since "bool(can_handle('foo://'))" will be True.
  return "Wrong source type"
  if get_archive_handler(self.base_url(source)):
  return True
@@ -106,7 +106,7 @@
  install_opener(opener)
  response = urlopen(source)
  try:
-- with open(dest, 'w') as dest_file:
++ with open(dest, 'wb') as dest_file:
  dest_file.write(response.read())
  except Exception as e:
  if os.path.isfile(dest):
@@ -155,7 +155,11 @@
  else:
  algorithms = hashlib.algorithms_available
  if key in algorithms:
-- check_hash(dld_file, value, key)
++ if len(value) != 1:
++ raise TypeError(
++ "Expected 1 hash value, not %d" % len(value))
++ expected = value[0]
++ check_hash(dld_file, expected, key)
  if checksum:
  check_hash(dld_file, checksum, hash_type)
  return extract(dld_file, dest)
 === modified file 'hooks/charmhelpers/fetch/bzrurl.py'
 --- hooks/charmhelpers/fetch/bzrurl.py	2015-02-03 18:59:19 +0000
 +++ hooks/charmhelpers/fetch/bzrurl.py	2016-09-16 19:19:06 +0000
@@ -1,78 +1,76 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
++from subprocess import check_call
  from charmhelpers.fetch import (
  BaseFetchHandler,
-- UnhandledSource
++ UnhandledSource,
++ filter_installed_packages,
++ install,
+ )
  from charmhelpers.core.host import mkdir
--import six
--if six.PY3:
-- raise ImportError('bzrlib does not support Python3')
--try:
-- from bzrlib.branch import Branch
-- from bzrlib import bzrdir, workingtree, errors
--except ImportError:
-- from charmhelpers.fetch import apt_install
-- apt_install("python-bzrlib")
-- from bzrlib.branch import Branch
-- from bzrlib import bzrdir, workingtree, errors
++if filter_installed_packages(['bzr']) != []:
++ install(['bzr'])
++ if filter_installed_packages(['bzr']) != []:
++ raise NotImplementedError('Unable to install bzr')
  class BzrUrlFetchHandler(BaseFetchHandler):
-- """Handler for bazaar branches via generic and lp URLs"""
++ """Handler for bazaar branches via generic and lp URLs."""
++
  def can_handle(self, source):
  url_parts = self.parse_url(source)
-- if url_parts.scheme not in ('bzr+ssh', 'lp'):
++ if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
  return False
++ elif not url_parts.scheme:
++ return os.path.exists(os.path.join(source, '.bzr'))
  else:
  return True
-- def branch(self, source, dest):
-- url_parts = self.parse_url(source)
-- # If we use lp:branchname scheme we need to load plugins
++ def branch(self, source, dest, revno=None):
  if not self.can_handle(source):
  raise UnhandledSource("Cannot handle {}".format(source))
-- if url_parts.scheme == "lp":
-- from bzrlib.plugin import load_plugins
-- load_plugins()
-- try:
-- local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
-- except errors.AlreadyControlDirError:
-- local_branch = Branch.open(dest)
-- try:
-- remote_branch = Branch.open(source)
-- remote_branch.push(local_branch)
-- tree = workingtree.WorkingTree.open(dest)
-- tree.update()
-- except Exception as e:
-- raise e
++ cmd_opts = []
++ if revno:
++ cmd_opts += ['-r', str(revno)]
++ if os.path.exists(dest):
++ cmd = ['bzr', 'pull']
++ cmd += cmd_opts
++ cmd += ['--overwrite', '-d', dest, source]
++ else:
++ cmd = ['bzr', 'branch']
++ cmd += cmd_opts
++ cmd += [source, dest]
++ check_call(cmd)
-- def install(self, source):
++ def install(self, source, dest=None, revno=None):
  url_parts = self.parse_url(source)
  branch_name = url_parts.path.strip("/").split("/")[-1]
-- dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
-- branch_name)
-- if not os.path.exists(dest_dir):
-- mkdir(dest_dir, perms=0o755)
++ if dest:
++ dest_dir = os.path.join(dest, branch_name)
++ else:
++ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
++ branch_name)
++
++ if dest and not os.path.exists(dest):
++ mkdir(dest, perms=0o755)
++
  try:
-- self.branch(source, dest_dir)
++ self.branch(source, dest_dir, revno)
  except OSError as e:
  raise UnhandledSource(e.strerror)
  return dest_dir
 === added file 'hooks/charmhelpers/fetch/centos.py'
 --- hooks/charmhelpers/fetch/centos.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/fetch/centos.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,171 @@
++# Copyright 2014-2015 Canonical Limited.
++#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
++
++import subprocess
++import os
++import time
++import six
++import yum
++
++from tempfile import NamedTemporaryFile
++from charmhelpers.core.hookenv import log
++
++YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM.
++YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
++YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
++
++
++def filter_installed_packages(packages):
++ """Return a list of packages that require installation."""
++ yb = yum.YumBase()
++ package_list = yb.doPackageLists()
++ temp_cache = {p.base_package_name: 1 for p in package_list['installed']}
++
++ _pkgs = [p for p in packages if not temp_cache.get(p, False)]
++ return _pkgs
++
++
++def install(packages, options=None, fatal=False):
++ """Install one or more packages."""
++ cmd = ['yum', '--assumeyes']
++ if options is not None:
++ cmd.extend(options)
++ cmd.append('install')
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++ log("Installing {} with options: {}".format(packages,
++ options))
++ _run_yum_command(cmd, fatal)
++
++
++def upgrade(options=None, fatal=False, dist=False):
++ """Upgrade all packages."""
++ cmd = ['yum', '--assumeyes']
++ if options is not None:
++ cmd.extend(options)
++ cmd.append('upgrade')
++ log("Upgrading with options: {}".format(options))
++ _run_yum_command(cmd, fatal)
++
++
++def update(fatal=False):
++ """Update local yum cache."""
++ cmd = ['yum', '--assumeyes', 'update']
++ log("Update with fatal: {}".format(fatal))
++ _run_yum_command(cmd, fatal)
++
++
++def purge(packages, fatal=False):
++ """Purge one or more packages."""
++ cmd = ['yum', '--assumeyes', 'remove']
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++ log("Purging {}".format(packages))
++ _run_yum_command(cmd, fatal)
++
++
++def yum_search(packages):
++ """Search for a package."""
++ output = {}
++ cmd = ['yum', 'search']
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++ log("Searching for {}".format(packages))
++ result = subprocess.check_output(cmd)
++ for package in list(packages):
++ output[package] = package in result
++ return output
++
++
++def add_source(source, key=None):
++ """Add a package source to this system.
++
++ @param source: a URL with a rpm package
++
++ @param key: A key to be added to the system's keyring and used
++ to verify the signatures on packages. Ideally, this should be an
++ ASCII format GPG public key including the block headers. A GPG key
++ id may also be used, but be aware that only insecure protocols are
++ available to retrieve the actual public key from a public keyserver
++ placing your Juju environment at risk.
++ """
++ if source is None:
++ log('Source is not present. Skipping')
++ return
++
++ if source.startswith('http'):
++ directory = '/etc/yum.repos.d/'
++ for filename in os.listdir(directory):
++ with open(directory + filename, 'r') as rpm_file:
++ if source in rpm_file.read():
++ break
++ else:
++ log("Add source: {!r}".format(source))
++ # write in the charms.repo
++ with open(directory + 'Charms.repo', 'a') as rpm_file:
++ rpm_file.write('[%s]\n' % source[7:].replace('/', '_'))
++ rpm_file.write('name=%s\n' % source[7:])
++ rpm_file.write('baseurl=%s\n\n' % source)
++ else:
++ log("Unknown source: {!r}".format(source))
++
++ if key:
++ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
++ with NamedTemporaryFile('w+') as key_file:
++ key_file.write(key)
++ key_file.flush()
++ key_file.seek(0)
++ subprocess.check_call(['rpm', '--import', key_file])
++ else:
++ subprocess.check_call(['rpm', '--import', key])
++
++
++def _run_yum_command(cmd, fatal=False):
++ """Run an YUM command.
++
++ Checks the output and retry if the fatal flag is set to True.
++
++ :param: cmd: str: The yum command to run.
++ :param: fatal: bool: Whether the command's output should be checked and
++ retried.
++ """
++ env = os.environ.copy()
++
++ if fatal:
++ retry_count = 0
++ result = None
++
++ # If the command is considered "fatal", we need to retry if the yum
++ # lock was not acquired.
++
++ while result is None or result == YUM_NO_LOCK:
++ try:
++ result = subprocess.check_call(cmd, env=env)
++ except subprocess.CalledProcessError as e:
++ retry_count = retry_count + 1
++ if retry_count > YUM_NO_LOCK_RETRY_COUNT:
++ raise
++ result = e.returncode
++ log("Couldn't acquire YUM lock. Will retry in {} seconds."
++ "".format(YUM_NO_LOCK_RETRY_DELAY))
++ time.sleep(YUM_NO_LOCK_RETRY_DELAY)
++
++ else:
++ subprocess.call(cmd, env=env)
 === modified file 'hooks/charmhelpers/fetch/giturl.py'
 --- hooks/charmhelpers/fetch/giturl.py	2015-07-14 14:05:42 +0000
 +++ hooks/charmhelpers/fetch/giturl.py	2016-09-16 19:19:06 +0000
@@ -1,58 +1,56 @@
  # Copyright 2014-2015 Canonical Limited.
+ #
--# This file is part of charm-helpers.
--#
--# charm-helpers is free software: you can redistribute it and/or modify
--# it under the terms of the GNU Lesser General Public License version 3 as
--# published by the Free Software Foundation.
--#
--# charm-helpers is distributed in the hope that it will be useful,
--# but WITHOUT ANY WARRANTY; without even the implied warranty of
--# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
--# GNU Lesser General Public License for more details.
--#
--# You should have received a copy of the GNU Lesser General Public License
--# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
  import os
++from subprocess import check_call, CalledProcessError
  from charmhelpers.fetch import (
  BaseFetchHandler,
-- UnhandledSource
++ UnhandledSource,
++ filter_installed_packages,
++ install,
+ )
--from charmhelpers.core.host import mkdir
--
--import six
--if six.PY3:
-- raise ImportError('GitPython does not support Python 3')
--
--try:
-- from git import Repo
--except ImportError:
-- from charmhelpers.fetch import apt_install
-- apt_install("python-git")
-- from git import Repo
--
--from git.exc import GitCommandError # noqa E402
++
++if filter_installed_packages(['git']) != []:
++ install(['git'])
++ if filter_installed_packages(['git']) != []:
++ raise NotImplementedError('Unable to install git')
  class GitUrlFetchHandler(BaseFetchHandler):
-- """Handler for git branches via generic and github URLs"""
++ """Handler for git branches via generic and github URLs."""
++
  def can_handle(self, source):
  url_parts = self.parse_url(source)
  # TODO (mattyw) no support for ssh git@ yet
-- if url_parts.scheme not in ('http', 'https', 'git'):
++ if url_parts.scheme not in ('http', 'https', 'git', ''):
  return False
++ elif not url_parts.scheme:
++ return os.path.exists(os.path.join(source, '.git'))
  else:
  return True
-- def clone(self, source, dest, branch, depth=None):
++ def clone(self, source, dest, branch="master", depth=None):
  if not self.can_handle(source):
  raise UnhandledSource("Cannot handle {}".format(source))
-- if depth:
-- Repo.clone_from(source, dest, branch=branch, depth=depth)
++ if os.path.exists(dest):
++ cmd = ['git', '-C', dest, 'pull', source, branch]
  else:
-- Repo.clone_from(source, dest, branch=branch)
++ cmd = ['git', 'clone', source, dest, '--branch', branch]
++ if depth:
++ cmd.extend(['--depth', depth])
++ check_call(cmd)
  def install(self, source, branch="master", dest=None, depth=None):
  url_parts = self.parse_url(source)
@@ -62,12 +60,10 @@
  else:
  dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
  branch_name)
-- if not os.path.exists(dest_dir):
-- mkdir(dest_dir, perms=0o755)
  try:
  self.clone(source, dest_dir, branch, depth)
-- except GitCommandError as e:
-- raise UnhandledSource(e.message)
++ except CalledProcessError as e:
++ raise UnhandledSource(e)
  except OSError as e:
  raise UnhandledSource(e.strerror)
  return dest_dir
 === added file 'hooks/charmhelpers/fetch/ubuntu.py'
 --- hooks/charmhelpers/fetch/ubuntu.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/fetch/ubuntu.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,313 @@
++# Copyright 2014-2015 Canonical Limited.
++#
++# Licensed under the Apache License, Version 2.0 (the "License");
++# you may not use this file except in compliance with the License.
++# You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing, software
++# distributed under the License is distributed on an "AS IS" BASIS,
++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++# See the License for the specific language governing permissions and
++# limitations under the License.
++
++import os
++import six
++import time
++import subprocess
++
++from tempfile import NamedTemporaryFile
++from charmhelpers.core.host import (
++ lsb_release
++)
++from charmhelpers.core.hookenv import log
++from charmhelpers.fetch import SourceConfigError
++
++CLOUD_ARCHIVE = ('# Ubuntu Cloud Archive deb'
++ ' http://ubuntu-cloud.archive.canonical.com/ubuntu'
++ ' {} main')
++PROPOSED_POCKET = ('# Proposed deb http://archive.ubuntu.com/ubuntu'
++ ' {}-proposed main universe multiverse restricted')
++CLOUD_ARCHIVE_POCKETS = {
++ # Folsom
++ 'folsom': 'precise-updates/folsom',
++ 'precise-folsom': 'precise-updates/folsom',
++ 'precise-folsom/updates': 'precise-updates/folsom',
++ 'precise-updates/folsom': 'precise-updates/folsom',
++ 'folsom/proposed': 'precise-proposed/folsom',
++ 'precise-folsom/proposed': 'precise-proposed/folsom',
++ 'precise-proposed/folsom': 'precise-proposed/folsom',
++ # Grizzly
++ 'grizzly': 'precise-updates/grizzly',
++ 'precise-grizzly': 'precise-updates/grizzly',
++ 'precise-grizzly/updates': 'precise-updates/grizzly',
++ 'precise-updates/grizzly': 'precise-updates/grizzly',
++ 'grizzly/proposed': 'precise-proposed/grizzly',
++ 'precise-grizzly/proposed': 'precise-proposed/grizzly',
++ 'precise-proposed/grizzly': 'precise-proposed/grizzly',
++ # Havana
++ 'havana': 'precise-updates/havana',
++ 'precise-havana': 'precise-updates/havana',
++ 'precise-havana/updates': 'precise-updates/havana',
++ 'precise-updates/havana': 'precise-updates/havana',
++ 'havana/proposed': 'precise-proposed/havana',
++ 'precise-havana/proposed': 'precise-proposed/havana',
++ 'precise-proposed/havana': 'precise-proposed/havana',
++ # Icehouse
++ 'icehouse': 'precise-updates/icehouse',
++ 'precise-icehouse': 'precise-updates/icehouse',
++ 'precise-icehouse/updates': 'precise-updates/icehouse',
++ 'precise-updates/icehouse': 'precise-updates/icehouse',
++ 'icehouse/proposed': 'precise-proposed/icehouse',
++ 'precise-icehouse/proposed': 'precise-proposed/icehouse',
++ 'precise-proposed/icehouse': 'precise-proposed/icehouse',
++ # Juno
++ 'juno': 'trusty-updates/juno',
++ 'trusty-juno': 'trusty-updates/juno',
++ 'trusty-juno/updates': 'trusty-updates/juno',
++ 'trusty-updates/juno': 'trusty-updates/juno',
++ 'juno/proposed': 'trusty-proposed/juno',
++ 'trusty-juno/proposed': 'trusty-proposed/juno',
++ 'trusty-proposed/juno': 'trusty-proposed/juno',
++ # Kilo
++ 'kilo': 'trusty-updates/kilo',
++ 'trusty-kilo': 'trusty-updates/kilo',
++ 'trusty-kilo/updates': 'trusty-updates/kilo',
++ 'trusty-updates/kilo': 'trusty-updates/kilo',
++ 'kilo/proposed': 'trusty-proposed/kilo',
++ 'trusty-kilo/proposed': 'trusty-proposed/kilo',
++ 'trusty-proposed/kilo': 'trusty-proposed/kilo',
++ # Liberty
++ 'liberty': 'trusty-updates/liberty',
++ 'trusty-liberty': 'trusty-updates/liberty',
++ 'trusty-liberty/updates': 'trusty-updates/liberty',
++ 'trusty-updates/liberty': 'trusty-updates/liberty',
++ 'liberty/proposed': 'trusty-proposed/liberty',
++ 'trusty-liberty/proposed': 'trusty-proposed/liberty',
++ 'trusty-proposed/liberty': 'trusty-proposed/liberty',
++ # Mitaka
++ 'mitaka': 'trusty-updates/mitaka',
++ 'trusty-mitaka': 'trusty-updates/mitaka',
++ 'trusty-mitaka/updates': 'trusty-updates/mitaka',
++ 'trusty-updates/mitaka': 'trusty-updates/mitaka',
++ 'mitaka/proposed': 'trusty-proposed/mitaka',
++ 'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
++ 'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
++ # Newton
++ 'newton': 'xenial-updates/newton',
++ 'xenial-newton': 'xenial-updates/newton',
++ 'xenial-newton/updates': 'xenial-updates/newton',
++ 'xenial-updates/newton': 'xenial-updates/newton',
++ 'newton/proposed': 'xenial-proposed/newton',
++ 'xenial-newton/proposed': 'xenial-proposed/newton',
++ 'xenial-proposed/newton': 'xenial-proposed/newton',
++}
++
++APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
++APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
++APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
++
++
++def filter_installed_packages(packages):
++ """Return a list of packages that require installation."""
++ cache = apt_cache()
++ _pkgs = []
++ for package in packages:
++ try:
++ p = cache[package]
++ p.current_ver or _pkgs.append(package)
++ except KeyError:
++ log('Package {} has no installation candidate.'.format(package),
++ level='WARNING')
++ _pkgs.append(package)
++ return _pkgs
++
++
++def apt_cache(in_memory=True, progress=None):
++ """Build and return an apt cache."""
++ from apt import apt_pkg
++ apt_pkg.init()
++ if in_memory:
++ apt_pkg.config.set("Dir::Cache::pkgcache", "")
++ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
++ return apt_pkg.Cache(progress)
++
++
++def install(packages, options=None, fatal=False):
++ """Install one or more packages."""
++ if options is None:
++ options = ['--option=Dpkg::Options::=--force-confold']
++
++ cmd = ['apt-get', '--assume-yes']
++ cmd.extend(options)
++ cmd.append('install')
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++ log("Installing {} with options: {}".format(packages,
++ options))
++ _run_apt_command(cmd, fatal)
++
++
++def upgrade(options=None, fatal=False, dist=False):
++ """Upgrade all packages."""
++ if options is None:
++ options = ['--option=Dpkg::Options::=--force-confold']
++
++ cmd = ['apt-get', '--assume-yes']
++ cmd.extend(options)
++ if dist:
++ cmd.append('dist-upgrade')
++ else:
++ cmd.append('upgrade')
++ log("Upgrading with options: {}".format(options))
++ _run_apt_command(cmd, fatal)
++
++
++def update(fatal=False):
++ """Update local apt cache."""
++ cmd = ['apt-get', 'update']
++ _run_apt_command(cmd, fatal)
++
++
++def purge(packages, fatal=False):
++ """Purge one or more packages."""
++ cmd = ['apt-get', '--assume-yes', 'purge']
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++ log("Purging {}".format(packages))
++ _run_apt_command(cmd, fatal)
++
++
++def apt_mark(packages, mark, fatal=False):
++ """Flag one or more packages using apt-mark."""
++ log("Marking {} as {}".format(packages, mark))
++ cmd = ['apt-mark', mark]
++ if isinstance(packages, six.string_types):
++ cmd.append(packages)
++ else:
++ cmd.extend(packages)
++
++ if fatal:
++ subprocess.check_call(cmd, universal_newlines=True)
++ else:
++ subprocess.call(cmd, universal_newlines=True)
++
++
++def apt_hold(packages, fatal=False):
++ return apt_mark(packages, 'hold', fatal=fatal)
++
++
++def apt_unhold(packages, fatal=False):
++ return apt_mark(packages, 'unhold', fatal=fatal)
++
++
++def add_source(source, key=None):
++ """Add a package source to this system.
++
++ @param source: a URL or sources.list entry, as supported by
++ add-apt-repository(1). Examples::
++
++ ppa:charmers/example
++ deb https://stub:key@private.example.com/ubuntu trusty main
++
++ In addition:
++ 'proposed:' may be used to enable the standard 'proposed'
++ pocket for the release.
++ 'cloud:' may be used to activate official cloud archive pockets,
++ such as 'cloud:icehouse'
++ 'distro' may be used as a noop
++
++ @param key: A key to be added to the system's APT keyring and used
++ to verify the signatures on packages. Ideally, this should be an
++ ASCII format GPG public key including the block headers. A GPG key
++ id may also be used, but be aware that only insecure protocols are
++ available to retrieve the actual public key from a public keyserver
++ placing your Juju environment at risk. ppa and cloud archive keys
++ are securely added automtically, so sould not be provided.
++ """
++ if source is None:
++ log('Source is not present. Skipping')
++ return
++
++ if (source.startswith('ppa:') or
++ source.startswith('http') or
++ source.startswith('deb ') or
++ source.startswith('cloud-archive:')):
++ subprocess.check_call(['add-apt-repository', '--yes', source])
++ elif source.startswith('cloud:'):
++ install(filter_installed_packages(['ubuntu-cloud-keyring']),
++ fatal=True)
++ pocket = source.split(':')[-1]
++ if pocket not in CLOUD_ARCHIVE_POCKETS:
++ raise SourceConfigError(
++ 'Unsupported cloud: source option %s' %
++ pocket)
++ actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
++ with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
++ apt.write(CLOUD_ARCHIVE.format(actual_pocket))
++ elif source == 'proposed':
++ release = lsb_release()['DISTRIB_CODENAME']
++ with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
++ apt.write(PROPOSED_POCKET.format(release))
++ elif source == 'distro':
++ pass
++ else:
++ log("Unknown source: {!r}".format(source))
++
++ if key:
++ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
++ with NamedTemporaryFile('w+') as key_file:
++ key_file.write(key)
++ key_file.flush()
++ key_file.seek(0)
++ subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
++ else:
++ # Note that hkp: is in no way a secure protocol. Using a
++ # GPG key id is pointless from a security POV unless you
++ # absolutely trust your network and DNS.
++ subprocess.check_call(['apt-key', 'adv', '--keyserver',
++ 'hkp://keyserver.ubuntu.com:80', '--recv',
++ key])
++
++
++def _run_apt_command(cmd, fatal=False):
++ """Run an APT command.
++
++ Checks the output and retries if the fatal flag is set
++ to True.
++
++ :param: cmd: str: The apt command to run.
++ :param: fatal: bool: Whether the command's output should be checked and
++ retried.
++ """
++ env = os.environ.copy()
++
++ if 'DEBIAN_FRONTEND' not in env:
++ env['DEBIAN_FRONTEND'] = 'noninteractive'
++
++ if fatal:
++ retry_count = 0
++ result = None
++
++ # If the command is considered "fatal", we need to retry if the apt
++ # lock was not acquired.
++
++ while result is None or result == APT_NO_LOCK:
++ try:
++ result = subprocess.check_call(cmd, env=env)
++ except subprocess.CalledProcessError as e:
++ retry_count = retry_count + 1
++ if retry_count > APT_NO_LOCK_RETRY_COUNT:
++ raise
++ result = e.returncode
++ log("Couldn't acquire DPKG lock. Will retry in {} seconds."
++ "".format(APT_NO_LOCK_RETRY_DELAY))
++ time.sleep(APT_NO_LOCK_RETRY_DELAY)
++
++ else:
++ subprocess.call(cmd, env=env)
 === added file 'hooks/charmhelpers/osplatform.py'
 --- hooks/charmhelpers/osplatform.py	1970-01-01 00:00:00 +0000
 +++ hooks/charmhelpers/osplatform.py	2016-09-16 19:19:06 +0000
@@ -0,0 +1,19 @@
++import platform
++
++
++def get_platform():
++ """Return the current OS platform.
++
++ For example: if current os platform is Ubuntu then a string "ubuntu"
++ will be returned (which is the name of the module).
++ This string is used to decide which platform module should be imported.
++ """
++ tuple_platform = platform.linux_distribution()
++ current_platform = tuple_platform[0]
++ if "Ubuntu" in current_platform:
++ return "ubuntu"
++ elif "CentOS" in current_platform:
++ return "centos"
++ else:
++ raise RuntimeError("This module is not supported on {}."
++ .format(current_platform))
 === modified file 'hooks/memcached_hooks.py'
 --- hooks/memcached_hooks.py	2016-09-02 09:04:58 +0000
 +++ hooks/memcached_hooks.py	2016-09-16 19:19:06 +0000
@@ -1,4 +1,4 @@
--#!/usr/bin/env python
++#!/usr/bin/env python3
  import re
  import os
  import shutil
 === added file 'setup.cfg'
 --- setup.cfg	1970-01-01 00:00:00 +0000
 +++ setup.cfg	2016-09-16 19:19:06 +0000
@@ -0,0 +1,5 @@
++[nosetests]
++verbosity=2
++with-coverage=1
++cover-erase=1
++cover-package=hooks
 === added file 'test-requirements-py2.txt'
 --- test-requirements-py2.txt	1970-01-01 00:00:00 +0000
 +++ test-requirements-py2.txt	2016-09-16 19:19:06 +0000
@@ -0,0 +1,9 @@
++nose==1.3.1
++Jinja2==2.7
++mock==1.0.1
++PyYAML==3.10
++six==1.5.2
++coverage==3.7.1
++flake8==2.1.0
++simplejson
++charm-tools
 === added file 'test-requirements-py3.txt'
 --- test-requirements-py3.txt	1970-01-01 00:00:00 +0000
 +++ test-requirements-py3.txt	2016-09-16 19:19:06 +0000
@@ -0,0 +1,7 @@
++nose==1.3.7
++Jinja2==2.8
++mock==1.3.0
++PyYAML==3.11
++six==1.10.0
++coverage==3.7.1
++flake8==2.5.4
 === added file 'tox.ini'
 --- tox.ini	1970-01-01 00:00:00 +0000
 +++ tox.ini	2016-09-16 19:19:06 +0000
@@ -0,0 +1,38 @@
++[tox]
++envlist = pep8,py27,py34,py35
++skipsdist = True
++
++[testenv]
++setenv = VIRTUAL_ENV={envdir}
++ PYTHONHASHSEED=0
++install_command =
++ pip install --allow-unverified python-apt {opts} {packages}
++commands = nosetests -s --nologcapture {posargs} unit_tests/
++
++# trusty
++[testenv:py27]
++basepython = python2.7
++deps = -r{toxinidir}/test-requirements-py2.txt
++
++# trusty
++[testenv:py34]
++basepython = python3.4
++deps = -r{toxinidir}/test-requirements-py3.txt
++
++# xenial
++[testenv:py35]
++basepython = python3.5
++deps = -r{toxinidir}/test-requirements-py3.txt
++
++[testenv:pep8]
++basepython = python2.7
++deps = -r{toxinidir}/test-requirements-py2.txt
++commands = flake8 {posargs} hooks unit_tests tests
++ charm-proof
++
++[testenv:venv]
++commands = {posargs}
++
++[flake8]
++ignore = E402,E226
++exclude = hooks/charmhelpers
 === modified file 'unit_tests/test_memcached_hooks.py'
 --- unit_tests/test_memcached_hooks.py	2015-07-22 20:13:58 +0000
 +++ unit_tests/test_memcached_hooks.py	2016-09-16 19:19:06 +0000
@@ -2,7 +2,7 @@
  import os
  import shutil
  import tempfile
--from test_utils import CharmTestCase
++from unit_tests.test_utils import CharmTestCase, get_default_config
  import memcached_hooks
  __author__ = 'Felipe Reyes <felipe.reyes@canonical.com>'
@@ -22,6 +22,7 @@
  'log',
  'oldest_peer',
  'peer_units',
++ 'open_port'
+ ]
  FREE_MEM_SMALL = """ total used free shared \
  buffers cached
@@ -159,23 +160,28 @@
  self.test_config.config['size'])
  open_port.assert_any_call(configs['tcp-port'], 'TCP')
++ @mock.patch('charmhelpers.contrib.network.ufw.modify_access')
++ @mock.patch('charmhelpers.contrib.network.ufw.grant_access')
++ @mock.patch('memcached_utils.config')
  @mock.patch('subprocess.Popen')
  @mock.patch('charmhelpers.core.templating.render')
  @mock.patch('subprocess.check_output')
  @mock.patch('memcached_utils.log')
  def test_config_changed_size_set_0_small(self, log, check_output, render,
-- popen):
++ popen, mucfg, *args):
  p = mock.Mock()
  p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'),
  'returncode': 0})
  popen.return_value = p
-- configs = {'size': 0}
++ configs = get_default_config()
++ configs.update({'size': 0})
  def f(c):
-- return configs.get(c, None)
++ return configs[c]
  self.config.side_effect = f
++ mucfg.side_effect = f
  def g(*args, **kwargs):
  if args[0] == ['free', '-m']:
@@ -191,30 +197,35 @@
  self.assertEqual(passed_vars['mem_size'], 1596)
  self.assertFalse(passed_vars['large_pages_enabled'])
++ @mock.patch('charmhelpers.contrib.network.ufw.modify_access')
++ @mock.patch('charmhelpers.contrib.network.ufw.grant_access')
++ @mock.patch('memcached_utils.config')
  @mock.patch('subprocess.Popen')
  @mock.patch('charmhelpers.core.templating.render')
  @mock.patch('subprocess.check_output')
  @mock.patch('memcached_utils.log')
  def test_config_changed_size_set_0_big(self, log, check_output, render,
-- popen):
++ popen, mucfg, *args):
  p = mock.Mock()
  p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'),
  'returncode': 0})
  popen.return_value = p
-- configs = {'size': 0,
-- 'disable-large-pages': False}
++ configs = get_default_config()
++ configs.update({'size': 0,
++ 'disable-large-pages': False})
  def f(c):
-- return configs.get(c, None)
++ return configs[c]
  self.config.side_effect = f
++ mucfg.side_effect = f
  def g(*args, **kwargs):
  if args[0] == ['free', '-m']:
  return FREE_MEM_BIG
  if args[0] == ['sysctl', '-n', 'vm.nr_hugepages']:
-- return int(16013 / 2)
++ return int(16013 * 2)
  else:
  return ""
@@ -228,24 +239,30 @@
  'vm.nr_hugepages={}'.format(16013/2)])
  self.assertTrue(passed_vars['large_pages_enabled'])
++ @mock.patch('charmhelpers.contrib.network.ufw.modify_access')
++ @mock.patch('charmhelpers.contrib.network.ufw.grant_access')
++ @mock.patch('memcached_utils.config')
  @mock.patch('subprocess.Popen')
  @mock.patch('charmhelpers.core.templating.render')
  @mock.patch('subprocess.check_output')
  @mock.patch('memcached_utils.log')
  def test_config_changed_size_failed_set_hpages(self, log, check_output,
-- render, popen):
++ render, popen, mucfg,
++ *args):
  p = mock.Mock()
  p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'),
  'returncode': 0})
  popen.return_value = p
-- configs = {'size': 0,
-- 'disable-large-pages': False}
++ configs = get_default_config()
++ configs.update({'size': 0,
++ 'disable-large-pages': False})
  def f(c):
-- return configs.get(c, None)
++ return configs[c]
  self.config.side_effect = f
++ mucfg.side_effect = f
  def g(*args, **kwargs):
  if args[0] == ['free', '-m']:
@@ -435,31 +452,31 @@
  cache_joined, ufw, unit_get, replica,
  open_port):
  params = {
-- 'tcp_port': None,
-- 'disable_cas': None,
-- 'mem_size': None,
-- 'factor': None,
-- 'connection_limit': None,
-- 'udp_port': None,
-- 'slab_page_size': None,
-- 'threads': None,
++ 'tcp_port': 11211,
++ 'disable_cas': False,
++ 'mem_size': 768,
++ 'factor': 1.25,
++ 'connection_limit': 1024,
++ 'udp_port': 0,
++ 'slab_page_size': -1,
++ 'threads': -1,
++ 'replica': "10.0.0.2",
++ 'repcached_port': '11212',
  'large_pages_enabled': False,
-- 'min_item_size': None,
-- 'repcached_port': '11212',
-- 'request_limit': None,
-- 'disable_auto_cleanup': None,
++ 'min_item_size': -1,
++ 'request_limit': -1,
++ 'disable_auto_cleanup': False,
+ }
  dpkg.return_value = True
  replica.return_value = ('10.0.0.2')
-- configs = {
-- 'repcached': True,
-- 'memsize': 1,
-- }
++ configs = get_default_config()
++ configs.update({'repcached': True,
++ 'memsize': 1})
  def f(c):
-- return configs.get(c, params.get(c))
++ return configs[c]
  self.config.side_effect = f
  self.unit_get.return_value = '10.0.0.1'
@@ -482,32 +499,31 @@
  def test_config_changed_no_replica(self, dpkg, render,
  replica, ufw, cache, open_port):
  params = {
-- 'tcp_port': None,
-- 'disable_cas': None,
-- 'mem_size': None,
-- 'factor': None,
-- 'connection_limit': None,
-- 'udp_port': None,
-- 'slab_page_size': None,
-- 'threads': None,
++ 'tcp_port': 11211,
++ 'disable_cas': False,
++ 'mem_size': 768,
++ 'factor': 1.25,
++ 'connection_limit': 1024,
++ 'udp_port': 0,
++ 'slab_page_size': -1,
++ 'threads': -1,
  'replica': "10.0.0.2",
  'repcached_port': '11212',
  'large_pages_enabled': False,
-- 'min_item_size': None,
-- 'request_limit': None,
-- 'disable_auto_cleanup': None,
++ 'min_item_size': -1,
++ 'request_limit': -1,
++ 'disable_auto_cleanup': False,
+ }
  replica.return_value = "10.0.0.2"
  dpkg.return_value = True
-- configs = {
-- 'repcached': True,
-- 'memsize': 1,
-- }
++ configs = get_default_config()
++ configs.update({'repcached': True,
++ 'memsize': 1})
  def f(c):
-- return configs.get(c, params.get(c))
++ return configs[c]
  self.config.side_effect = f
 === modified file 'unit_tests/test_utils.py'
 --- unit_tests/test_utils.py	2014-12-01 19:11:26 +0000
 +++ unit_tests/test_utils.py	2016-09-16 19:19:06 +0000
@@ -37,7 +37,7 @@
  '''
  default_config = {}
  config = load_config()
-- for k, v in config.iteritems():
++ for k, v in config.items():
  if 'default' in v:
  default_config[k] = v['default']
  else:

Juju Charms Collectionmemcached package

Merge lp:~freyes/charms/trusty/memcached/py3-tox into lp:charms/trusty/memcached

Commit message

Description of the change

Preview Diff

Subscribers

Juju Charms Collection
memcached package