summaryrefslogtreecommitdiff
diff options
authorStuart Bishop <stuart.bishop@canonical.com>2018-05-28 14:23:18 +0700
committerStuart Bishop <stuart.bishop@canonical.com>2018-05-28 14:23:18 +0700
commit8faa36725b36c34ca625ea88391420083b3a2934 (patch)
tree3c23aa793da2eb5bdb79f2b8cea09c0fe11f2b58
parent001d9b56d9597c8b20dc3b5b8cf7927a12a243b2 (diff)
parent44d66f8b955b26606cf04f3f6fb7ddfc478d5e14 (diff)
Merge branch 'status-fix' of git+ssh://git.launchpad.net/~freyes/mongodb-charm
-rwxr-xr-xhooks/hooks.py146
l---------hooks/update-status1
-rw-r--r--tests/base_deploy.py2
-rw-r--r--tests/deploy_replicaset.py32
-rw-r--r--unit_tests/test_hooks.py30
5 files changed, 175 insertions, 36 deletions
diff --git a/hooks/hooks.py b/hooks/hooks.py
index d3774a7..9de334d 100755
--- a/hooks/hooks.py
+++ b/hooks/hooks.py
@@ -6,7 +6,9 @@ Created on Aug 1, 2012
'''
import commands
+import json
import os
+import pprint
import re
import signal
import socket
@@ -30,14 +32,15 @@ from string import Template
from textwrap import dedent
from yaml.constructor import ConstructorError
+from charmhelpers.core.decorators import retry_on_exception
+from charmhelpers.payload.execd import execd_preinstall
+
from charmhelpers.fetch import (
add_source,
apt_update,
apt_install
)
-import json
-
from charmhelpers.core.host import (
service,
lsb_release,
@@ -47,6 +50,7 @@ from charmhelpers.core.hookenv import (
close_port,
config,
is_relation_made,
+ log as juju_log,
open_port,
unit_get,
relation_get,
@@ -58,15 +62,12 @@ from charmhelpers.core.hookenv import (
Hooks,
DEBUG,
WARNING,
- is_leader
+ is_leader,
+ status_set,
+ application_version_set,
)
-from charmhelpers.core.hookenv import log as juju_log
-
-from charmhelpers.payload.execd import execd_preinstall
-
from charmhelpers.contrib.hahelpers.cluster import (
- oldest_peer,
peer_units
)
@@ -586,8 +587,8 @@ def remove_replset_from_upstart():
"""
try:
mongodb_init_config = open(default_mongodb_init_config).read()
-
- if re.search(' --replSet', mongodb_init_config,
+
+ if re.search(' --replSet', mongodb_init_config,
re.MULTILINE) is not None:
mongodb_init_config = re.sub(' --replSet .\w+', '',
mongodb_init_config)
@@ -921,6 +922,7 @@ def arm64_trusty_quirk():
def install_hook():
juju_log('Begin install hook.')
execd_preinstall()
+ status_set('maintenance', 'Installing packages')
juju_log("Installing mongodb")
add_source(config('source'), config('key'))
@@ -935,9 +937,9 @@ def install_hook():
@hooks.hook('config-changed')
def config_changed():
juju_log("Entering config_changed")
- print "Entering config_changed"
+ status_set('maintenance', 'Configuring unit')
config_data = config()
- print "config_data: ", config_data
+ juju_log("config_data: {}".format(config_data), level=DEBUG)
mongodb_config = open(default_mongodb_config).read()
# Trigger volume initialization logic for permanent storage
@@ -976,11 +978,12 @@ def config_changed():
current_web_admin_ui_port = int(current_mongodb_port) + 1000
new_web_admin_ui_port = int(config_data['port']) + 1000
- print "current_mongodb_port: ", current_mongodb_port
+ juju_log("Configured mongodb port: {}".format(current_mongodb_port),
+ level=DEBUG)
public_address = unit_get('public-address')
- print "public_address: ", public_address
+ juju_log("unit's public_address: {}".format(public_address), level=DEBUG)
private_address = unit_get('private-address')
- print "private_address: ", private_address
+ juju_log("unit's private_address: {}".format(private_address), level=DEBUG)
# Update mongodb configuration file
mongodb_config = mongodb_conf(config_data)
@@ -1009,6 +1012,7 @@ def config_changed():
write_logrotate_config(config_data)
# restart mongodb
+ status_set('maintenance', 'Restarting mongod')
restart_mongod()
# attach to replSet ( if needed )
@@ -1069,8 +1073,10 @@ def config_changed():
open_port(config_data['mongos_port'])
update_nrpe_config()
+ application_version_set(get_mongod_version())
+ update_status()
- print "About to leave config_changed"
+ juju_log("About to leave config_changed", level=DEBUG)
return(True)
@@ -1148,6 +1154,7 @@ def replica_set_relation_joined():
if enable_replset(my_replset):
juju_log('Restarting mongodb after config change (enable replset)',
level=DEBUG)
+ status_set('maintenance', 'Restarting mongod to enable replicaset')
restart_mongod()
relation_set(relation_id(), {
@@ -1157,6 +1164,8 @@ def replica_set_relation_joined():
'install-order': my_install_order,
'type': 'replset',
})
+
+ update_status()
juju_log("replica_set_relation_joined-finish")
@@ -1165,7 +1174,8 @@ def am_i_primary():
for i in xrange(10):
try:
r = run_admin_command(c, 'replSetGetStatus')
- juju_log('am_i_primary: replSetGetStatus returned: %s' % str(r),
+ pretty_r = pprint.pformat(r)
+ juju_log('am_i_primary: replSetGetStatus returned: %s' % pretty_r,
level=DEBUG)
return r['myState'] == MONGO_PRIMARY
except OperationFailure as e:
@@ -1180,7 +1190,7 @@ def am_i_primary():
# replication not initialized yet (Mongo3.4+)
return False
elif 'not running with --replSet' in str(e):
- # replicaset not configured
+ # replicaset not configured
return False
else:
raise
@@ -1191,6 +1201,56 @@ def am_i_primary():
raise TimeoutException('Unable to determine if local unit is primary')
+def get_replicaset_status():
+ """Connect to mongod and get the status of replicaset
+ This function is used mainly within update_status() to display
+ replicaset status in 'juju status' output
+
+ :returns string: can be any of replicaset states
+ (https://docs.mongodb.com/manual/reference/replica-states/)
+ or can be the string of an exception while getting the status
+ """
+
+ c = MongoClient('localhost')
+ try:
+ r = run_admin_command(c, 'replSetGetStatus')
+ for member in r['members']:
+ if 'self' in member:
+ return member['stateStr']
+
+ # if 'self' was not found in the output, then log a warning and print
+ # the output given by replSetGetStatus
+ r_pretty = pprint.pformat(r)
+ juju_log('get_replicaset_status() failed to get replicaset state:' +
+ r_pretty, level=WARNING)
+ return 'Unknown replica set state'
+
+ except OperationFailure as e:
+ juju_log('get_replicaset_status() exception: %s' % str(e), DEBUG)
+ if 'not running with --replSet' in str(e):
+ return 'not in replicaset'
+ else:
+ return str(e)
+
+def get_mongod_version():
+ """ Connects to mongod and get the db.version() output
+ Mainly used for application_set_version in config-changed hook
+ """
+
+ c = MongoClient('localhost')
+ return c.server_info()['version']
+
+
+# Retry until the replica set is in active state, retry 45 times before failing,
+# wait 1, 2, 3, 4, ... seconds between each retry, this will add 33 minutes of
+# accumulated sleep()
+@retry_on_exception(num_retries=45, base_delay=1)
+def wait_until_replset_is_active():
+ status = update_status()
+ if status != 'active':
+ raise Exception('ReplicaSet not active: {}'.format(status))
+
+
@hooks.hook('replica-set-relation-changed')
def replica_set_relation_changed():
private_address = unit_get('private-address')
@@ -1208,6 +1268,7 @@ def replica_set_relation_changed():
# Initialize the replicaset - we do this only on the leader!
if is_leader():
juju_log('Initializing replicaset')
+ status_set('maintenance', 'Initializing replicaset')
init_replset()
unit = "%s:%s" % (private_address, config('port'))
@@ -1218,9 +1279,11 @@ def replica_set_relation_changed():
juju_log('Adding new secondary... %s' % unit_remote, level=DEBUG)
join_replset(unit, unit_remote)
+ wait_until_replset_is_active()
juju_log('replica_set_relation_changed-finish')
+
@hooks.hook('replica-set-relation-departed')
def replica_set_relation_departed():
juju_log('replica_set_relation_departed-start')
@@ -1263,12 +1326,12 @@ def replica_set_relation_broken():
c = MongoClient('localhost')
r = c.admin.command('isMaster')
-
+
try:
master_node = r['primary']
except KeyError:
pass
-
+
if 'master_node' in locals(): # unit is part of replicaset, remove it!
unit = "%s:%s" % (unit_get('private-address'), config('port'))
juju_log('Removing myself via %s' % (master_node), 'DEBUG')
@@ -1350,11 +1413,12 @@ def mongos_relation_changed():
port = relation_get('port')
rel_type = relation_get('type')
if hostname is None or port is None or rel_type is None:
- print("mongos_relation_changed: relation data not ready.")
+ juju_log("mongos_relation_changed: relation data not ready.",
+ level=DEBUG)
return
if rel_type == 'configsvr':
config_servers = load_config_servers(default_mongos_list)
- print "Adding config server: %s:%s" % (hostname, port)
+ juju_log("Adding config server: %s:%s" % (hostname, port), level=DEBUG)
if hostname is not None and \
port is not None and \
hostname != '' and \
@@ -1379,11 +1443,11 @@ def mongos_relation_changed():
mongo_client(mongos_host, shard_command2)
else:
- print("mongos_relation_change: undefined rel_type: %s" %
- rel_type)
+ juju_log("mongos_relation_change: undefined rel_type: %s" % rel_type,
+ level=DEBUG)
return
- print("mongos_relation_changed returns: %s" % retVal)
+ juju_log("mongos_relation_changed returns: %s" % retVal, level=DEBUG)
@hooks.hook('mongos-relation-broken')
@@ -1439,6 +1503,38 @@ def uprade_charm():
remove_rest_from_upstart()
+@hooks.hook('update-status')
+def update_status():
+ """
+ Returns: workload_state (so that some hooks know they need to re-run
+ update_status if needed)
+ """
+
+ workload = 'active'
+ status = 'Unit is ready'
+
+ if is_relation_made('replica-set'):
+ # only check for replica-set state if the relation was made which means
+ # more than 1 units were deployed and peer related.
+ mongo_status = get_replicaset_status()
+ if mongo_status in ('PRIMARY', 'SECONDARY'):
+ workload = 'active'
+ status = 'Unit is ready as ' + mongo_status
+ elif mongo_status in ('not in replicaset',):
+ workload = 'active'
+ status = 'Unit is ready, ' + mongo_status
+ else:
+ workload = 'maintenance'
+ status = mongo_status
+ juju_log('mongo_status is unknown: {}'.format(status), level=DEBUG)
+
+ juju_log('Setting workload: {} - {}'.format(workload, status), level=DEBUG)
+ status_set(workload, status)
+
+ return workload
+
+
+
def run(command, exit_on_error=True):
'''Run a command and return the output.'''
try:
diff --git a/hooks/update-status b/hooks/update-status
new file mode 120000
index 0000000..9416ca6
--- /dev/null
+++ b/hooks/update-status
@@ -0,0 +1 @@
+hooks.py \ No newline at end of file
diff --git a/tests/base_deploy.py b/tests/base_deploy.py
index 8930966..b136cca 100644
--- a/tests/base_deploy.py
+++ b/tests/base_deploy.py
@@ -24,8 +24,6 @@ class BasicMongo(object):
message = 'The environment did not setup in %d seconds.',
self.deploy_timeout
amulet.raise_status(amulet.SKIP, msg=message)
- except:
- raise
self.sentry_dict = {svc: self.d.sentry[svc]
for svc in list(self.d.sentry.unit)}
diff --git a/tests/deploy_replicaset.py b/tests/deploy_replicaset.py
index 9506407..6fa0290 100644
--- a/tests/deploy_replicaset.py
+++ b/tests/deploy_replicaset.py
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
import amulet
+import logging
+import re
import sys
import time
import traceback
@@ -12,6 +14,7 @@ from base_deploy import BasicMongo
# max amount of time to wait before testing for replicaset status
wait_for_replicaset = 600
+logger = logging.getLogger(__name__)
class Replicaset(BasicMongo):
@@ -109,6 +112,34 @@ class Replicaset(BasicMongo):
def validate_running_services(self):
super(Replicaset, self).validate_running_services()
+ def validate_workload_status(self):
+ primaries = 0
+ secondaries = 0
+ regex = re.compile('^Unit is ready as (PRIMARY|SECONDARY)$')
+ self.d.sentry.wait_for_messages({'mongodb': regex})
+
+ # count how many primaries and secondaries were reported in the
+ # workload status
+ for unit_name, unit in self.d.sentry.get_status()['mongodb'].items():
+ workload_msg = unit['workload-status']['message']
+ matched = re.match(regex, workload_msg)
+
+ if not matched:
+ msg = "'{}' does not match '{}'".format(workload_msg, regex)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ elif matched.group(1) == 'PRIMARY':
+ primaries += 1
+ elif matched.group(1) == 'SECONDARY':
+ secondaries += 1
+ else:
+ amulet.raise_status(amulet.FAIL,
+ msg='Unknown state: %s' % matched.group(1))
+
+ logger.debug('Secondary units found: %d' % secondaries)
+ if primaries > 1:
+ msg = "Found %d primaries, expected 1" % primaries
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
def run(self):
self.deploy()
self.validate_status_interface()
@@ -116,3 +147,4 @@ class Replicaset(BasicMongo):
self.validate_replicaset_setup()
self.validate_replicaset_relation_joined()
self.validate_world_connectivity()
+ self.validate_workload_status()
diff --git a/unit_tests/test_hooks.py b/unit_tests/test_hooks.py
index 603b149..354d104 100644
--- a/unit_tests/test_hooks.py
+++ b/unit_tests/test_hooks.py
@@ -36,19 +36,24 @@ class MongoHooksTest(CharmTestCase):
self.config.side_effect = self.test_config.get
self.relation_get.side_effect = self.test_relation.get
+ @patch.object(hooks, 'is_relation_made')
+ @patch.object(hooks, 'get_replicaset_status')
@patch.object(hooks, 'restart_mongod')
@patch.object(hooks, 'enable_replset')
# Note: patching the os.environ dictionary in-line here so there's no
# additional parameter sent into the function
@patch.dict('os.environ', JUJU_UNIT_NAME='fake-unit/0')
def test_replica_set_relation_joined(self, mock_enable_replset,
- mock_restart):
+ mock_restart, mock_get_replset_status,
+ mock_is_rel_made):
self.unit_get.return_value = 'private.address'
self.test_config.set('port', '1234')
self.test_config.set('replicaset', 'fake-replicaset')
self.relation_id.return_value = 'fake-relation-id'
mock_enable_replset.return_value = False
+ mock_get_replset_status.return_value = 'PRIMARY'
+ mock_is_rel_made.return_value = True
hooks.replica_set_relation_joined()
@@ -276,17 +281,24 @@ class MongoHooksTest(CharmTestCase):
mock_subprocess.assert_called_once_with(expected_cmd, shell=True)
self.assertFalse(rv)
+ @patch.object(hooks, 'is_relation_made')
+ @patch.object(hooks, 'run_admin_command')
+ @patch.object(hooks, 'is_leader')
+ @patch.object(hooks, 'get_replicaset_status')
@patch.object(hooks, 'am_i_primary')
@patch.object(hooks, 'init_replset')
@patch.object(hooks, 'relation_get')
@patch.object(hooks, 'peer_units')
- @patch.object(hooks, 'oldest_peer')
@patch.object(hooks, 'join_replset')
@patch.object(hooks, 'unit_get')
def test_replica_set_relation_changed(self, mock_unit_get,
- mock_join_replset, mock_oldest_peer,
- mock_peer_units, mock_relation_get,
- mock_init_replset, mock_is_primary):
+ mock_join_replset, mock_peer_units,
+ mock_relation_get, mock_init_replset,
+ mock_is_primary,
+ mock_get_replset_status,
+ mock_is_leader,
+ mock_run_admin_cmd,
+ mock_is_rel_made):
# set the unit_get('private-address')
mock_unit_get.return_value = 'juju-local-unit-0.local'
mock_relation_get.return_value = None
@@ -298,6 +310,10 @@ class MongoHooksTest(CharmTestCase):
# Test remote hostname is valid, but master is somehow not defined
mock_join_replset.reset_mock()
mock_relation_get.return_value = 'juju-local-unit-0'
+ mock_is_leader.return_value = False
+ mock_run_admin_cmd.return_value = {'myState': hooks.MONGO_PRIMARY}
+ mock_get_replset_status.return_value = 'PRIMARY'
+ mock_is_rel_made.return_value = True
hooks.replica_set_relation_changed()
@@ -306,9 +322,7 @@ class MongoHooksTest(CharmTestCase):
# Test when not oldest peer, don't init replica set
mock_join_replset.reset_mock()
mock_init_replset.reset_mock()
- mock_oldest_peer.reset_mock()
mock_peer_units.return_value = ['mongodb/1', 'mongodb/2']
- mock_oldest_peer.return_value = False
hooks.replica_set_relation_changed()
@@ -317,8 +331,6 @@ class MongoHooksTest(CharmTestCase):
# Test when its also the PRIMARY
mock_relation_get.reset_mock()
mock_relation_get.side_effect = ['juju-remote-unit-0', '12345']
- mock_oldest_peer.reset_mock()
- mock_oldest_peer.return_value = False
mock_is_primary.reset_mock()
mock_is_primary.return_value = True
mock_join_replset.reset_mock()