Add checks for replica setsstable/21.04stable/21.01

Reviewed-on: https://code.launchpad.net/~peter-sabaini/charm-mongodb/+git/charm-mongodb/+merge/396580 Reviewed-by: Xav Paice <xav.paice@canonical.com> Reviewed-by: Drew Freiberger <drew.freiberger@canonical.com> 

8 files changed, 410 insertions, 2 deletions

diff --git a/files/nrpe-external-master/check_replica_sets.py b/files/nrpe-external-master/check_replica_sets.py
new file mode 100755
index 0000000..f6d8feb
--- /dev/null
+++ b/files/nrpe-external-master/check_replica_sets.py
@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+"""NRPE check to verify if mongodb replicas are healthy."""
+
+#
+# Copyright 2020 Canonical Ltd.
+#
+# Author: David O Neill <david.o.neill@canonical.com>
+#
+
+import json
+import subprocess
+import sys
+
+import yaml
+
+NAGIOS_OK = 0
+NAGIOS_WARN = 1
+NAGIOS_CRIT = 2
+NAGIOS_UNKNOWN = 3
+
+REPL_STATES_OK = ["PRIMARY", "SECONDARY", "ARBITER"]
+# Unused, but for reference:
+# REPL_STATES_WARN = ["STARTUP", "STARTUP2", "ROLLBACK", "RECOVERING"]
+REPL_STATES_CRIT = ["UNKNOWN", "DOWN", "REMOVED"]
+
+
+def get_cmd_result(cmd):
+ try:
+ return subprocess.check_output(cmd).decode("utf8")
+ except subprocess.CalledProcessError as e:
+ print("CRITICAL: failed executing {}, error was {}".format(cmd, e))
+ sys.exit(NAGIOS_CRIT)
+
+
+def get_replica_set_name():
+ # mongo2 file format the key is 'replSet'
+ # mongo34 yaml file format the key is 'replSetName'
+ # this will match both
+
+ with open("/etc/mongodb.conf", "r") as mongo_file:
+ data = mongo_file.read()
+ try:
+ config = yaml.safe_load(data)
+ except yaml.scanner.ScannerError:
+ config = None
+
+ if isinstance(config, dict):
+ if "replication" not in config:
+ return False
+ if "replSetName" not in config["replication"]:
+ return False
+ return config["replication"]["replSetName"]
+
+ for line in data.splitlines():
+ if line.strip().startswith("replSet") is False:
+ continue
+ parts = line.split("=")
+ return parts[1].strip()
+
+ return False
+
+
+def get_replica_status():
+ # https://bugs.launchpad.net/charm-mongodb/+bug/1892122
+ # Looking for RECOVERING
+ # {
+ # "members" : [
+ # {
+ # ...
+ # "stateStr" : "SECONDARY",
+ # ...
+ # },
+ # ...
+ # ]
+ # }
+
+ name = get_replica_set_name()
+ if not name:
+ print("CRITIAL: unable to determine the replica set name")
+ sys.exit(NAGIOS_CRIT)
+
+ mongo_res = get_cmd_result(
+ ["/usr/bin/mongo", name, "--quiet", "--eval", "JSON.stringify(rs.status())"]
+ )
+
+ try:
+ res_json = json.loads(mongo_res)
+ except json.decoder.JSONDecodeError:
+ print("CRITICAL: failed to load json")
+ sys.exit(NAGIOS_CRIT)
+
+ warnings = []
+ errors = []
+
+ for member in res_json["members"]:
+ state = member["stateStr"]
+ if state in REPL_STATES_OK:
+ continue
+ elif state in REPL_STATES_CRIT:
+ errors.append("{}: {}".format(member["name"], state))
+ else:
+ warnings.append("{}: {}".format(member["name"], state))
+
+ if errors:
+ print(
+ "CRITICAL: the following members are in error states: {}".format(
+ ", ".join(errors)
+ )
+ )
+ sys.exit(NAGIOS_CRIT)
+
+ if warnings:
+ print(
+ "WARN: the following members are in warning states: {}".format(
+ ", ".join(warnings)
+ )
+ )
+ sys.exit(NAGIOS_WARN)
+
+ print("OK")
+ sys.exit(NAGIOS_OK)
+
+
+if __name__ == "__main__":
+ get_replica_status()
diff --git a/hooks/hooks.py b/hooks/hooks.py
index 5963db6..254d11b 100755
--- a/hooks/hooks.py
+++ b/hooks/hooks.py
@@ -38,6 +38,7 @@ from charmhelpers.core.hookenv import (
 UnregisteredHookError,
 WARNING,
 application_version_set,
+ charm_dir,
 close_port,
 config,
 env_proxy_settings,
@@ -57,6 +58,7 @@ from charmhelpers.core.hookenv import (
 from charmhelpers.core.host import (
 file_hash,
 lsb_release,
+ rsync,
 service,
 )
 from charmhelpers.fetch import add_source, apt_install, apt_update
@@ -1497,7 +1499,10 @@ def update_nrpe_config():
 # Find out if nrpe set nagios_hostname
 hostname = None
 host_context = None
- for rel in relations_of_type("nrpe-external-master"):
+ reldata = relations_of_type("nrpe-external-master")
+ if not reldata:
+ return
+ for rel in reldata:
 if "nagios_hostname" in rel:
 hostname = rel["nagios_hostname"]
 host_context = rel["nagios_host_context"]
@@ -1521,6 +1526,17 @@ def update_nrpe_config():
 check_cmd=check_mongo_script,
 )
 
+ charm_plugin_dir = os.path.join(charm_dir(), "files", "nrpe-external-master/")
+ rsync(
+ charm_plugin_dir, "/usr/local/lib/nagios/plugins/", options=["--executability"]
+ )
+
+ nrpe.add_check(
+ shortname="check_replica_sets",
+ description="replica set check {%s}" % current_unit,
+ check_cmd="check_replica_sets.py",
+ )
+
 nrpe.write()
 
 
diff --git a/hooks/nrpe-external-master-relation-departed b/hooks/nrpe-external-master-relation-departed
new file mode 120000
index 0000000..9416ca6
--- /dev/null
+++ b/hooks/nrpe-external-master-relation-departed
@@ -0,0 +1 @@
+hooks.py \ No newline at end of file
diff --git a/tests/functional/tests/bundles/focal-nrpe.yaml b/tests/functional/tests/bundles/focal-nrpe.yaml
new file mode 100644
index 0000000..7cb8fa8
--- /dev/null
+++ b/tests/functional/tests/bundles/focal-nrpe.yaml
@@ -0,0 +1,18 @@
+series: focal
+description: "mongodb-charm test bundle"
+applications:
+ mongodb:
+ num_units: 3
+ options:
+ replicaset: testset
+ backup_directory: /var/backups
+ nrpe:
+ charm: cs:nrpe
+ nagios:
+ series: bionic
+ charm: cs:nagios
+ num_units: 1
+
+relations:
+ - [mongodb, nrpe]
+ - [nagios:monitors, nrpe]
diff --git a/tests/functional/tests/tests.yaml b/tests/functional/tests/tests.yaml
index 6461753..d67bc5c 100644
--- a/tests/functional/tests/tests.yaml
+++ b/tests/functional/tests/tests.yaml
@@ -16,6 +16,7 @@ gate_bundles:
 - model_alias_xenial: xenial
 - model_alias_bionic: bionic
 - model_alias_focal: focal
+ - model_alias_focal: focal-nrpe
 # - model_alias_shard: bionic-shard
 smoke_bundles:
 - model_alias_bionic: bionic
diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt
index aaa5368..55e3ba6 100644
--- a/tests/unit/requirements.txt
+++ b/tests/unit/requirements.txt
@@ -6,3 +6,4 @@ charm-tools
 packaging
 appdirs
 pymongo
+PyYAML
diff --git a/tests/unit/test_check_replica_sets.py b/tests/unit/test_check_replica_sets.py
new file mode 100644
index 0000000..5eb9b6e
--- /dev/null
+++ b/tests/unit/test_check_replica_sets.py
@@ -0,0 +1,246 @@
+"""Test check_replica_sets.py nrpe check."""
+import unittest
+import unittest.mock as mock
+
+from check_replica_sets import get_cmd_result, get_replica_set_name, get_replica_status
+
+import yaml
+
+try:
+ from StringIO import StringIO # for Python 2
+except ImportError:
+ from io import StringIO # for Python 3
+
+
+class TestCheckReplicaSets(unittest.TestCase):
+ """Test class."""
+
+ @mock.patch("check_replica_sets.subprocess.check_output", autospec=True)
+ def test_get_cmd_result_true(self, mock_check_output):
+ """Test result of subprocess call."""
+ mock_check_output.return_value = b"1"
+ output = get_cmd_result("echo 1")
+ self.assertEqual(output, "1")
+
+ @mock.patch("check_replica_sets.subprocess.check_output", autospec=True)
+ def test_get_cmd_result_false(self, mock_check_output):
+ """Test error cmd via subprocess call."""
+ result = "CRITICAL: failed executing {cmd}, error was {error}".format(
+ cmd="dont care", error="Boom!"
+ )
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_cmd_result("dont care")
+ except Exception:
+ self.assertEqual(fake_std_out.getvalue(), result)
+
+ def test_get_replica_set_name_yaml_replsetname(self):
+ """Test mongodb.conf file parsing replSetName success."""
+ sample = {}
+ sample["replication"] = {}
+ sample["replication"]["replSetName"] = "test"
+ sample_yaml = yaml.dump(sample)
+
+ with mock.patch(
+ "builtins.open", mock.mock_open(read_data=sample_yaml)
+ ) as mock_file:
+ res = get_replica_set_name()
+ mock_file.assert_called_with("/etc/mongodb.conf", "r")
+ self.assertEqual(res, "test")
+
+ def test_get_replica_set_name_yaml_no_replsetname(self):
+ """Test mongodb.conf file parsing fail, where there's no replSetName."""
+ sample = {}
+ sample["replication"] = {}
+ sample_yaml = yaml.dump(sample)
+
+ with mock.patch(
+ "builtins.open", mock.mock_open(read_data=sample_yaml)
+ ) as mock_file:
+ res = get_replica_set_name()
+ mock_file.assert_called_with("/etc/mongodb.conf", "r")
+ self.assertEqual(res, False)
+
+ def test_get_replica_set_name_yaml_no_replication(self):
+ """Test mongodb.conf file parsing fail, with no replication block."""
+ sample = {}
+ sample["somethingelse"] = {}
+ sample_yaml = yaml.dump(sample)
+
+ with mock.patch(
+ "builtins.open", mock.mock_open(read_data=sample_yaml)
+ ) as mock_file:
+ res = get_replica_set_name()
+ mock_file.assert_called_with("/etc/mongodb.conf", "r")
+ self.assertEqual(res, False)
+
+ def test_get_replica_set_name_ini_true(self):
+ """Test mongodb.conf file parsing success in ini format."""
+ sample = """
+ # mongodb.conf
+ dbpath\t=\t/var/lib/mongodb # tab added to make it fail when parsing as YAML
+ ipv6=true
+ logpath=/var/log/mongodb/mongodb.log
+ logappend=true
+ port = 27017
+ journal=true
+ diaglog = 0
+ rest = true
+ replSet = test
+ """
+
+ with mock.patch("builtins.open", mock.mock_open(read_data=sample)) as mock_file:
+ res = get_replica_set_name()
+ mock_file.assert_called_with("/etc/mongodb.conf", "r")
+ self.assertEqual(res, "test")
+
+ def test_get_replica_set_name_ini_no_replset(self):
+ """Test mongodb.conf file missing replSet in ini format."""
+ sample = """
+ # mongodb.conf
+ dbpath=/var/lib/mongodb
+ ipv6=true
+ logpath=/var/log/mongodb/mongodb.log
+ logappend=true
+ port = 27017
+ journal=true
+ diaglog = 0
+ rest = true
+ """
+
+ with mock.patch("builtins.open", mock.mock_open(read_data=sample)) as mock_file:
+ res = get_replica_set_name()
+ mock_file.assert_called_with("/etc/mongodb.conf", "r")
+ self.assertEqual(res, False)
+
+ @mock.patch("check_replica_sets.get_replica_set_name", autospec=True)
+ @mock.patch("check_replica_sets.get_cmd_result", autospec=True)
+ def test_get_replica_status_ok(
+ self, mock_get_cmd_result, mock_get_replica_set_name
+ ):
+ """Test replica status in known good states gives OK."""
+ mock_get_replica_set_name.return_value = "test"
+ mock_get_cmd_result.return_value = """{
+ "set" : "myset",
+ "members" : [
+ {
+ "stateStr" : "SECONDARY"
+ },
+ {
+ "stateStr" : "PRIMARY"
+ },
+ {
+ "stateStr" : "SECONDARY"
+ }
+ ]
+ }"""
+
+ result = "OK\n"
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_replica_status()
+ except BaseException as error:
+ self.assertEqual(fake_std_out.getvalue(), result)
+ self.assertEqual("0", str(error))
+
+ @mock.patch("check_replica_sets.get_replica_set_name", autospec=True)
+ @mock.patch("check_replica_sets.get_cmd_result", autospec=True)
+ def test_get_replica_status_recovering_warn(
+ self, mock_get_cmd_result, mock_get_replica_set_name
+ ):
+ """Test replica status in RECOVERING state gives WARN."""
+ mock_get_replica_set_name.return_value = "test"
+ mock_get_cmd_result.return_value = """{
+ "set" : "myset",
+ "members" : [
+ {
+ "name": "one",
+ "stateStr" : "SECONDARY"
+ },
+ {
+ "name": "two",
+ "stateStr" : "RECOVERING"
+ },
+ {
+ "name": "three",
+ "stateStr" : "RECOVERING"
+ }
+ ]
+ }"""
+
+ result = (
+ "WARN: the following members are in warning states: "
+ "two: RECOVERING, three: RECOVERING\n"
+ )
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_replica_status()
+ except BaseException as error:
+ self.assertEqual(fake_std_out.getvalue(), result)
+ self.assertEqual("1", str(error))
+
+ @mock.patch("check_replica_sets.get_replica_set_name", autospec=True)
+ @mock.patch("check_replica_sets.get_cmd_result", autospec=True)
+ def test_get_replica_status_down_crit(
+ self, mock_get_cmd_result, mock_get_replica_set_name
+ ):
+ """Test replica status in DOWN state gives CRIT."""
+ mock_get_replica_set_name.return_value = "test"
+ mock_get_cmd_result.return_value = """{
+ "set" : "myset",
+ "members" : [
+ {
+ "name": "one",
+ "stateStr" : "SECONDARY"
+ },
+ {
+ "name": "two",
+ "stateStr" : "PRIMARY"
+ },
+ {
+ "name": "three",
+ "stateStr" : "DOWN"
+ }
+ ]
+ }"""
+
+ result = "CRITICAL: the following members are in error states: " "three: DOWN\n"
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_replica_status()
+ except BaseException as error:
+ self.assertEqual(fake_std_out.getvalue(), result)
+ self.assertEqual("2", str(error))
+
+ @mock.patch("check_replica_sets.get_replica_set_name", autospec=True)
+ @mock.patch("check_replica_sets.get_cmd_result", autospec=True)
+ def test_get_replica_status_no_replicasetname_crit(
+ self, mock_get_cmd_result, mock_get_replica_set_name
+ ):
+ """Test rs.status() failure 1."""
+ mock_get_replica_set_name.return_value = False
+
+ result = "CRITIAL: unable to determine the replica set name\n"
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_replica_status()
+ except BaseException as error:
+ self.assertEqual(fake_std_out.getvalue(), result)
+ self.assertEqual("2", str(error))
+
+ @mock.patch("check_replica_sets.get_replica_set_name", autospec=True)
+ @mock.patch("check_replica_sets.get_cmd_result", autospec=True)
+ def test_get_replica_status_bad_json_crit(
+ self, mock_get_cmd_result, mock_get_replica_set_name
+ ):
+ """Test rs.status() failure 2."""
+ mock_get_replica_set_name.return_value = "dont care"
+ mock_get_cmd_result.return_value = "{ this json is bogus }"
+
+ result = "CRITICAL: failed to load json\n"
+ with mock.patch("sys.stdout", new=StringIO()) as fake_std_out:
+ try:
+ get_replica_status()
+ except BaseException as error:
+ self.assertEqual(fake_std_out.getvalue(), result)
+ self.assertEqual("2", str(error))
diff --git a/tox.ini b/tox.ini
index 20097ae..f1baafa 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,7 +6,7 @@ envlist = lint, unit, func
 [testenv]
 basepython = python3
 setenv =
- PYTHONPATH = {toxinidir}:{toxinidir}/lib/:{toxinidir}/hooks/
+ PYTHONPATH = {toxinidir}:{toxinidir}/lib/:{toxinidir}/hooks/:{toxinidir}/files/nrpe-external-master/
 passenv =
 HOME
 PATH