diff options
| -rw-r--r-- | .jujuignore | 1 | ||||
| -rw-r--r-- | actions/backup.py | 13 | ||||
| -rw-r--r-- | actions/backup_test.py | 22 | ||||
| -rwxr-xr-x | hooks/hooks.py | 1156 | ||||
| -rw-r--r-- | tests/functional/tests/tests_mongodb.py | 3 | ||||
| -rw-r--r-- | tests/unit/__init__.py | 5 | ||||
| -rw-r--r-- | tests/unit/test_hooks.py | 398 | ||||
| -rw-r--r-- | tests/unit/test_utils.py | 27 | ||||
| -rw-r--r-- | tests/unit/test_write_log_rotate_config.py | 25 | ||||
| -rw-r--r-- | tox.ini | 7 |
10 files changed, 852 insertions, 805 deletions
diff --git a/.jujuignore b/.jujuignore deleted file mode 100644 index b7f1399..0000000 --- a/.jujuignore +++ /dev/null @@ -1 +0,0 @@ -.build \ No newline at end of file diff --git a/actions/backup.py b/actions/backup.py index 0930eab..a5682be 100644 --- a/actions/backup.py +++ b/actions/backup.py @@ -1,11 +1,12 @@ import shlex import subprocess import os + try: from charmhelpers.core.hookenv import action_get, action_set, action_fail except ImportError: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-pip']) - subprocess.check_call(['pip3', 'install', 'charmhelpers']) + subprocess.check_call(["apt-get", "install", "-y", "python3-pip"]) + subprocess.check_call(["pip3", "install", "charmhelpers"]) from charmhelpers.core.hookenv import action_get, action_set, action_fail @@ -14,10 +15,7 @@ def mkdir(dir): def execute(command, current_working_directory): - return subprocess.check_output( - shlex.split(command), - cwd=current_working_directory, - ) + return subprocess.check_output(shlex.split(command), cwd=current_working_directory) def dump(): @@ -49,8 +47,7 @@ def backup_command(cmd, args, dir): output = execute(command, dir) action_set({"output": output}) except subprocess.CalledProcessError as e: - action_set({"error_code": e.returncode, - "exception": e, "output": e.output}) + action_set({"error_code": e.returncode, "exception": e, "output": e.output}) action_fail(str(e)) except Exception as e: action_set({"exception": e}) diff --git a/actions/backup_test.py b/actions/backup_test.py index b5f1d33..916024e 100644 --- a/actions/backup_test.py +++ b/actions/backup_test.py @@ -36,19 +36,27 @@ class TestBackups(unittest.TestCase): backup.mkdir.assert_called_once_with("this/dir") backup.execute.assert_called_once_with("mongodump -arg1 -arg2", "this/dir") - backup.action_set.assert_has_calls([ - call({"command": "mongodump -arg1 -arg2", "working-dir": "this/dir"}), - call({"output": "output"})]) + backup.action_set.assert_has_calls( + [ + call({"command": "mongodump -arg1 -arg2", "working-dir": "this/dir"}), + call({"output": "output"}), + ] + ) def test_restore(self): backup.restore() backup.mkdir.assert_called_once_with("this/dir") backup.execute.assert_called_once_with("mongorestore -arg1 -arg2", "this/dir") - backup.action_set.assert_has_calls([ - call({"command": "mongorestore -arg1 -arg2", "working-dir": "this/dir"}), - call({"output": "output"})]) + backup.action_set.assert_has_calls( + [ + call( + {"command": "mongorestore -arg1 -arg2", "working-dir": "this/dir"} + ), + call({"output": "output"}), + ] + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/hooks/hooks.py b/hooks/hooks.py index 0c7798b..4ba26fb 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -''' +""" Created on Aug 1, 2012 @author: negronjl -''' +""" import collections import distutils @@ -21,9 +21,9 @@ try: import yaml # flake8: noqa except ImportError: if sys.version_info.major == 2: - subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) + subprocess.check_call(["apt-get", "install", "-y", "python-yaml"]) else: - subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + subprocess.check_call(["apt-get", "install", "-y", "python3-yaml"]) import yaml from os import chmod @@ -36,11 +36,7 @@ from yaml.constructor import ConstructorError from charmhelpers.core.decorators import retry_on_exception from charmhelpers.payload.execd import execd_preinstall -from charmhelpers.fetch import ( - add_source, - apt_update, - apt_install -) +from charmhelpers.fetch import add_source, apt_update, apt_install from charmhelpers.core.host import ( service, @@ -70,9 +66,7 @@ from charmhelpers.core.hookenv import ( application_version_set, ) -from charmhelpers.contrib.hahelpers.cluster import ( - peer_units -) +from charmhelpers.contrib.hahelpers.cluster import peer_units try: from pymongo import MongoClient @@ -88,13 +82,13 @@ except ImportError: try: import pip # flake8: noqa except ImportError: - apt_install('python3-pip', fatal=True) + apt_install("python3-pip", fatal=True) import pip # flake8: noqa try: import distro # flake8: noqa except ImportError: - pip.main(['install', "distro"]) + pip.main(["install", "distro"]) import distro # flake8: noqa from charmhelpers.contrib.charmsupport.nrpe import NRPE @@ -111,7 +105,7 @@ default_mongos_list = "/etc/mongos.list" default_wait_for = 3 default_max_tries = 7 -INSTALL_PACKAGES = ['mongodb-server', 'python3-yaml'] +INSTALL_PACKAGES = ["mongodb-server", "python3-yaml"] # number of seconds init_replset will pause while looping to check if # replicaset is initialized @@ -143,32 +137,32 @@ was_i_primary = False def is_bionic_or_greater(): current_version = distro.linux_distribution()[1] - if distutils.version.LooseVersion(current_version) >= \ - distutils.version.LooseVersion('18.04'): + if distutils.version.LooseVersion( + current_version + ) >= distutils.version.LooseVersion("18.04"): return True -def port_check(host=None, port=None, protocol='TCP'): +def port_check(host=None, port=None, protocol="TCP"): if host is None or port is None: juju_log("port_check: host and port must be defined.") - return(False) - if protocol.upper() == 'TCP': + return False + if protocol.upper() == "TCP": s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - elif protocol.upper() == 'UDP': + elif protocol.upper() == "UDP": s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) else: juju_log("port_check: Unrecognized protocol %s" % protocol) - return(False) + return False try: s.connect((host, int(port))) s.shutdown(socket.SHUT_RDWR) juju_log("port_check: %s:%s/%s is open" % (host, port, protocol)) - return(True) + return True except Exception as e: - juju_log("port_check: Unable to connect to %s:%s/%s." % - (host, port, protocol)) + juju_log("port_check: Unable to connect to %s:%s/%s." % (host, port, protocol)) juju_log("port_check: Exception: %s" % str(e)) - return(False) + return False # update_service_ports: Convenience function that evaluate the old and new @@ -177,7 +171,7 @@ def port_check(host=None, port=None, protocol='TCP'): def update_service_ports(old_service_ports=None, new_service_ports=None): juju_log("update_service_ports") if old_service_ports is None or new_service_ports is None: - return(None) + return None for port in old_service_ports: if port not in new_service_ports: juju_log("closing port: %d" % int(port)) @@ -197,7 +191,7 @@ def regex_sub(pat_replace=None, data=None): new_data = data for (pattern, replace) in pat_replace: new_data = re.sub(pattern, replace, data, 0, re.MULTILINE) - return(new_data) + return new_data def update_file(filename=None, new_data=None, old_data=None): @@ -206,21 +200,22 @@ def update_file(filename=None, new_data=None, old_data=None): retVal = False try: if old_data != new_data: - with open(filename, 'w') as f: + with open(filename, "w") as f: f.write(new_data) retVal = True except Exception as e: juju_log(str(e)) retVal = False finally: - return(retVal) + return retVal def process_check(pid=None): try: if pid is not None: - cmd_line = subprocess.check_output('ps -p %d -o cmd h' % - int(pid), shell=True) + cmd_line = subprocess.check_output( + "ps -p %d -o cmd h" % int(pid), shell=True + ) retVal = (pid, cmd_line) else: juju_log("process_check: pid not defined.") @@ -229,22 +224,20 @@ def process_check(pid=None): juju_log("process_check exception: %s" % str(e)) retVal = (None, None) finally: - juju_log("process_check returs pid: %s and cmd_line: %s" % - retVal) - return(retVal) + juju_log("process_check returs pid: %s and cmd_line: %s" % retVal) + return retVal def process_check_pidfile(pidfile=None): if pidfile is not None and os.path.exists(pidfile): - return(process_check(open(pidfile).read())) + return process_check(open(pidfile).read()) else: juju_log("process_check_pidfile: undefined or non-existant pidfile.") - return((None, None)) + return (None, None) def is_valid_ip(bind_ip): - if bind_ip == unit_get("private-address") or \ - bind_ip == unit_get("public-address"): + if bind_ip == unit_get("private-address") or bind_ip == unit_get("public-address"): return True return False @@ -252,12 +245,14 @@ def is_valid_ip(bind_ip): def choose_bind_ip(bind_ip): default_value, localhost = "0.0.0.0", "127.0.0.1" known_values = collections.defaultdict(lambda: False) - known_values.update({ - "0.0.0.0": default_value, - "all": default_value, - "localhost": localhost, - "127.0.0.1": localhost, - }) + known_values.update( + { + "0.0.0.0": default_value, + "all": default_value, + "localhost": localhost, + "127.0.0.1": localhost, + } + ) if known_values[bind_ip]: return known_values[bind_ip] @@ -282,7 +277,7 @@ class TimeoutException(Exception): ############################################################################### def mongodb_conf(config_data=None): # noqa: C901 if config_data is None: - return(None) + return None config = [] # header @@ -291,28 +286,25 @@ def mongodb_conf(config_data=None): # noqa: C901 # dbpath # Create the directory if not there already - subprocess.call(['mkdir', '-p', '%s' % config_data['dbpath']]) + subprocess.call(["mkdir", "-p", "%s" % config_data["dbpath"]]) # Make sure the mongodb user has access to it - subprocess.call(['chown', '-R', 'mongodb:mongodb', config_data['dbpath']]) - config.append("dbpath=%s" % config_data['dbpath']) + subprocess.call(["chown", "-R", "mongodb:mongodb", config_data["dbpath"]]) + config.append("dbpath=%s" % config_data["dbpath"]) config.append("") config.append("ipv6=true") # logpath # Create the directory if not there already - subprocess.call(['mkdir', - '-p', - '%s' % os.path.dirname(config_data['logpath'])]) - subprocess.call(['chown', - '-R', - 'mongodb:mongodb', - os.path.dirname(config_data['logpath'])]) - - config.append("logpath=%s" % config_data['logpath']) + subprocess.call(["mkdir", "-p", "%s" % os.path.dirname(config_data["logpath"])]) + subprocess.call( + ["chown", "-R", "mongodb:mongodb", os.path.dirname(config_data["logpath"])] + ) + + config.append("logpath=%s" % config_data["logpath"]) config.append("") # log_append - if config_data['logappend']: + if config_data["logappend"]: config.append("logappend=true") config.append("") @@ -322,67 +314,67 @@ def mongodb_conf(config_data=None): # noqa: C901 config.append("") # port - config.append("port = %d" % config_data['port']) + config.append("port = %d" % config_data["port"]) config.append("") # journal - if config_data['journal']: + if config_data["journal"]: config.append("journal=true") config.append("") # cpu - if config_data['cpu']: + if config_data["cpu"]: config.append("cpu = true") config.append("") # auth - if config_data['auth']: + if config_data["auth"]: config.append("auth = true") config.append("") # verbose - if config_data['verbose']: + if config_data["verbose"]: config.append("verbose = true") config.append("") # objcheck - if config_data['objcheck']: + if config_data["objcheck"]: config.append("objcheck = true") config.append("") # quota - if config_data['quota']: + if config_data["quota"]: config.append("quota = true") - config.append("quotaFiles = {}".format(config_data['quotafiles'])) + config.append("quotaFiles = {}".format(config_data["quotafiles"])) config.append("") if not is_bionic_or_greater(): # diaglog - config.append("diaglog = %d" % config_data['diaglog']) + config.append("diaglog = %d" % config_data["diaglog"]) config.append("") # nohttpinterface - if config_data['web_admin_ui']: + if config_data["web_admin_ui"]: config.append("rest = true") config.append("") # noscripting - if config_data['noscripting']: + if config_data["noscripting"]: config.append("noscripting = true") config.append("") # notablescan - if config_data['notablescan']: + if config_data["notablescan"]: config.append("notablescan = true") config.append("") # noprealloc - if config_data['noprealloc']: + if config_data["noprealloc"]: config.append("noprealloc = true") config.append("") # nssize - if config_data['nssize'] != "default": - config.append("nssize = %s" % config_data['nssize']) + if config_data["nssize"] != "default": + config.append("nssize = %s" % config_data["nssize"]) config.append("") # Set either replica-set or master, depending upon whether the @@ -391,40 +383,39 @@ def mongodb_conf(config_data=None): # noqa: C901 # charm will use replica-set replication. The user may opt to # do master/slave replication or sharding as a different form # of scaleout. - if is_relation_made('replica-set'): - config.append("replSet = %s" % config_data['replicaset']) + if is_relation_made("replica-set"): + config.append("replSet = %s" % config_data["replicaset"]) config.append("") else: - if config_data['master'] == "self": + if config_data["master"] == "self": config.append("master = true") config.append("") else: config.append("slave = true") - config.append("source = %s" % config_data['master']) + config.append("source = %s" % config_data["master"]) config.append("") # arbiter - if config_data['arbiter'] != "disabled" and \ - config_data['arbiter'] != "enabled": - config.append("arbiter = %s" % config_data['arbiter']) + if config_data["arbiter"] != "disabled" and config_data["arbiter"] != "enabled": + config.append("arbiter = %s" % config_data["arbiter"]) config.append("") # autoresync - if config_data['autoresync']: + if config_data["autoresync"]: config.append("autoresync") config.append("") # oplogSize - if config_data['oplogSize'] != "default": - config.append("oplogSize = %s" % config_data['oplogSize']) + if config_data["oplogSize"] != "default": + config.append("oplogSize = %s" % config_data["oplogSize"]) config.append("") # extra config options - if config_data['extra_config_options'] != "none": - for config_option in config_data['extra_config_options'].split(','): + if config_data["extra_config_options"] != "none": + for config_option in config_data["extra_config_options"].split(","): config.append(config_option) - return('\n'.join(config)) + return "\n".join(config) def get_current_mongo_config(): @@ -434,65 +425,61 @@ def get_current_mongo_config(): :returns dict: key/value pairs of the configuration file. """ results = {} - with open(default_mongodb_config, 'r') as f: + with open(default_mongodb_config, "r") as f: for line in f: line = line.strip() # Skip over comments, blank lines, and any other line # that appears to not contain a key = value pair. - if line.startswith('#') or '=' not in line: + if line.startswith("#") or "=" not in line: continue - key, value = line.split('=', 1) + key, value = line.split("=", 1) results[key.strip()] = value.strip() return results def mongo_client(host=None, command=None): if host is None or command is None: - return(False) + return False else: - cmd_line = 'mongo' - cmd_line += ' --host %s' % host - cmd_line += ' --eval \'printjson(%s)\'' % command + cmd_line = "mongo" + cmd_line += " --host %s" % host + cmd_line += " --eval 'printjson(%s)'" % command juju_log("Executing: %s" % cmd_line, level=DEBUG) - return(subprocess.call(cmd_line, shell=True) == 0) + return subprocess.call(cmd_line, shell=True) == 0 -def mongo_client_smart(host='localhost', command=None): - ''' +def mongo_client_smart(host="localhost", command=None): + """ Rework of mongo_client function, it retries the command MONGO_CLIENT_RETRIES times :param host: The host to connect to. Defaults to localhost :param command: The command to be executed. Can't be None :returns True if command succeeded, False if it failed - ''' + """ if command is None: return False - cmd_line = ['mongo', '--quiet', '--host', host, - '--eval', 'printjson(%s)' % command] + cmd_line = ["mongo", "--quiet", "--host", host, "--eval", "printjson(%s)" % command] juju_log("mongo_client_smart executing: %s" % str(cmd_line), level=DEBUG) for i in range(MONGO_CLIENT_RETRIES): try: cmd_output = subprocess.check_output(cmd_line).decode("utf8") - juju_log('mongo_client_smart executed, output: %s' % - cmd_output) - if json.loads(cmd_output)['ok'] == 1: + juju_log("mongo_client_smart executed, output: %s" % cmd_output) + if json.loads(cmd_output)["ok"] == 1: return True except subprocess.CalledProcessError as err: - juju_log('mongo_client_smart failed: %s' % - err.output, - level=DEBUG) + juju_log("mongo_client_smart failed: %s" % err.output, level=DEBUG) pass finally: time.sleep(1.5) # At this point, the command failed - juju_log('mongo_client_smart failed executing: %s', level=WARNING) + juju_log("mongo_client_smart failed executing: %s", level=WARNING) return False @@ -501,46 +488,44 @@ def init_replset(): # Use my IP at rs.initiate(), voids issues with invalid (and/or # not resolvable by peers) hostnames - rset = config_data['replicaset'] - addr = unit_get('private-address') - port = config_data['port'] + rset = config_data["replicaset"] + addr = unit_get("private-address") + port = config_data["port"] - init = '{_id: "%s", members: [{_id: 0, host: "%s:%s"}]} ' % (rset, - addr, - port) + init = '{_id: "%s", members: [{_id: 0, host: "%s:%s"}]} ' % (rset, addr, port) - retVal = mongo_client('localhost', 'rs.initiate(%s)' % init) + retVal = mongo_client("localhost", "rs.initiate(%s)" % init) time.sleep(1) # give mongod some time to become primary - c = MongoClient('localhost') + c = MongoClient("localhost") while True: try: - r = run_admin_command(c, 'replSetGetStatus') - mongo_state = r['myState'] - juju_log('init_replset: myState: %s' % mongo_state) + r = run_admin_command(c, "replSetGetStatus") + mongo_state = r["myState"] + juju_log("init_replset: myState: %s" % mongo_state) if mongo_state == MONGO_PRIMARY: # we're primary! break - elif mongo_state in (MONGO_STARTUP, - MONGO_STARTUP2, - MONGO_SECONDARY - ): # we are still initializing + elif mongo_state in ( + MONGO_STARTUP, + MONGO_STARTUP2, + MONGO_SECONDARY, + ): # we are still initializing continue else: - juju_log('init_replset: Unexpected replicaSet state: %s' % - mongo_state) + juju_log("init_replset: Unexpected replicaSet state: %s" % mongo_state) retVal = False break except OperationFailure as e: - juju_log('init_replset: OperationFailure: %s' % e, DEBUG) - if 'Received replSetInitiate' in str(e): + juju_log("init_replset: OperationFailure: %s" % e, DEBUG) + if "Received replSetInitiate" in str(e): continue else: - juju_log('init_replset: Unhandled OperationFailure %s' % e) + juju_log("init_replset: Unhandled OperationFailure %s" % e) raise finally: time.sleep(INIT_CHECK_DELAY) juju_log("init_replset returns: %s" % retVal, level=DEBUG) - return(retVal) + return retVal def run_admin_command(client, cmdstr): @@ -556,49 +541,49 @@ def join_replset(master_node=None, host=None): # localhost. # However, that might break other code calling this method. # This will wait charm rewrite. - juju_log("join_replset: master_node: %s, host: %s" % - (master_node, host)) + juju_log("join_replset: master_node: %s, host: %s" % (master_node, host)) if master_node is None or host is None: retVal = False else: retVal = rs_add(host) juju_log("join_replset returns: %s" % retVal, level=DEBUG) - return(retVal) + return retVal def leave_replset(master_node=None, host=None): - juju_log("leave_replset: master_node: %s, host: %s" % - (master_node, host)) + juju_log("leave_replset: master_node: %s, host: %s" % (master_node, host)) if master_node is None or host is None: retVal = False else: retVal = mongo_client(master_node, 'rs.remove("%s")' % host) juju_log("leave_replset returns: %s" % retVal, level=DEBUG) - return(retVal) + return retVal def enable_replset(replicaset_name=None): retVal = False if replicaset_name is None: - juju_log('enable_replset: replicaset_name is None, exiting', - level=DEBUG) + juju_log("enable_replset: replicaset_name is None, exiting", level=DEBUG) try: - juju_log('enable_replset: Enabling replicaset configuration:') + juju_log("enable_replset: Enabling replicaset configuration:") current_config = get_current_mongo_config() config_data = dict(config()) - if 'replSet' in current_config and \ - current_config['replSet'] == config_data['replicaset']: - juju_log('enable_replset: replica set is already enabled', - level=DEBUG) + if ( + "replSet" in current_config + and current_config["replSet"] == config_data["replicaset"] + ): + juju_log("enable_replset: replica set is already enabled", level=DEBUG) else: - juju_log('enable_replset: enabling replicaset %s' % - config_data['replicaset'], level=DEBUG) + juju_log( + "enable_replset: enabling replicaset %s" % config_data["replicaset"], + level=DEBUG, + ) mongodb_config = mongodb_conf(config_data) retVal = update_file(default_mongodb_config, mongodb_config) - juju_log('enable_replset will return: %s' % str(retVal), level=DEBUG) + juju_log("enable_replset will return: %s" % str(retVal), level=DEBUG) except Exception as e: juju_log(str(e), level=WARNING) @@ -613,32 +598,29 @@ def remove_replset_from_upstart(): try: mongodb_init_config = open(default_mongodb_init_config).read() - if re.search(' --replSet', mongodb_init_config, - re.MULTILINE) is not None: - mongodb_init_config = re.sub(r' --replSet .\w+', '', - mongodb_init_config) + if re.search(" --replSet", mongodb_init_config, re.MULTILINE) is not None: + mongodb_init_config = re.sub(r" --replSet .\w+", "", mongodb_init_config) retVal = update_file(default_mongodb_init_config, mongodb_init_config) except Exception as e: juju_log(str(e)) retVal = False finally: - return(retVal) + return retVal def step_down_replset_primary(): """Steps down the primary """ - retVal = mongo_client('localhost', 'rs.stepDown()') + retVal = mongo_client("localhost", "rs.stepDown()") for i in range(MONGO_CLIENT_RETRIES): if not am_i_primary(): - juju_log("step_down_replset_primary returns: %s" % retVal, - level=DEBUG) - return(retVal) + juju_log("step_down_replset_primary returns: %s" % retVal, level=DEBUG) + return retVal # If we are still primary wait a bit more for step down time.sleep(1.5) # Raise an error if we exhausted the maximum amount of trials - raise TimeoutException('Unable to step down the primary') + raise TimeoutException("Unable to step down the primary") def remove_rest_from_upstart(): @@ -646,23 +628,21 @@ def remove_rest_from_upstart(): """ try: mongodb_init_config = open(default_mongodb_init_config).read() - if re.search(' --rest ', mongodb_init_config, - re.MULTILINE) is not None: - mongodb_init_config = regex_sub([(' --rest ', ' ')], - mongodb_init_config) + if re.search(" --rest ", mongodb_init_config, re.MULTILINE) is not None: + mongodb_init_config = regex_sub([(" --rest ", " ")], mongodb_init_config) retVal = update_file(default_mongodb_init_config, mongodb_init_config) except Exception as e: juju_log(str(e)) retVal = False finally: - return(retVal) + return retVal def update_daemon_options(daemon_options=None): if is_bionic_or_greater(): if daemon_options and daemon_options != "none": daemon_opts = 'DAEMON_OPTS="{0}"\n'.format(daemon_options) - return(update_file(mongodb_env_config, daemon_opts)) + return update_file(mongodb_env_config, daemon_opts) else: if os.path.exists(mongodb_env_config): os.remove(mongodb_env_config) @@ -672,95 +652,93 @@ def update_daemon_options(daemon_options=None): pat_replace = [] if daemon_options is None or daemon_options == "none": pat_replace.append( - (' --config /etc/mongodb.conf.*', - ' --config /etc/mongodb.conf; fi')) + (" --config /etc/mongodb.conf.*", " --config /etc/mongodb.conf; fi") + ) else: pat_replace.append( - (' --config /etc/mongodb.conf.*', - ' --config /etc/mongodb.conf %s; fi' % daemon_options)) + ( + " --config /etc/mongodb.conf.*", + " --config /etc/mongodb.conf %s; fi" % daemon_options, + ) + ) regex_sub(pat_replace, mongodb_init_config) - return(update_file(default_mongodb_init_config, mongodb_init_config)) + return update_file(default_mongodb_init_config, mongodb_init_config) def enable_arbiter(master_node=None, host=None): - juju_log("enable_arbiter: master_node: %s, host: %s" % - (master_node, host)) + juju_log("enable_arbiter: master_node: %s, host: %s" % (master_node, host)) if master_node is None or host is None: retVal = False else: - retVal = mongo_client(master_node, "rs.addArb(\"%s\")" % host) + retVal = mongo_client(master_node, 'rs.addArb("%s")' % host) juju_log("enable_arbiter returns: %s" % retVal) - return(retVal) + return retVal def configsvr_status(wait_for=default_wait_for, max_tries=default_max_tries): config_data = config() current_try = 0 - while (process_check_pidfile('/var/run/mongodb/configsvr.pid') != ( - None, None)) and not port_check( - unit_get('private-address'), - config_data['config_server_port']) and current_try < max_tries: + while ( + (process_check_pidfile("/var/run/mongodb/configsvr.pid") != (None, None)) + and not port_check( + unit_get("private-address"), config_data["config_server_port"] + ) + and current_try < max_tries + ): juju_log("configsvr_status: Waiting for Config Server to be ready ...") time.sleep(wait_for) current_try += 1 - retVal = (process_check_pidfile('/var/run/mongodb/configsvr.pid') != (None, None) - ) == port_check(unit_get('private-address'), - config_data['config_server_port']) is True + retVal = ( + (process_check_pidfile("/var/run/mongodb/configsvr.pid") != (None, None)) + == port_check(unit_get("private-address"), config_data["config_server_port"]) + is True + ) if retVal: - return(process_check_pidfile('/var/run/mongodb/configsvr.pid')) + return process_check_pidfile("/var/run/mongodb/configsvr.pid") else: - return((None, None)) + return (None, None) def configsvr_ready(wait_for=default_wait_for, max_tries=default_max_tries): - return(configsvr_status(wait_for, max_tries) != (None, None)) + return configsvr_status(wait_for, max_tries) != (None, None) def disable_configsvr(port=None): if port is None: juju_log("disable_configsvr: port not defined.") - return(False) + return False try: - config_server_port = config('config_server_port') - pid = open('/var/run/mongodb/configsvr.pid').read() + config_server_port = config("config_server_port") + pid = open("/var/run/mongodb/configsvr.pid").read() os.kill(int(pid), signal.SIGTERM) - os.unlink('/var/run/mongodb/configsvr.pid') + os.unlink("/var/run/mongodb/configsvr.pid") retVal = True except Exception as e: - juju_log('no config server running ...') + juju_log("no config server running ...") juju_log("Exception: %s" % str(e)) retVal = False finally: juju_log("disable_configsvr returns %s" % retVal) close_port(config_server_port) - return(retVal) + return retVal -def enable_configsvr(config_data, wait_for=default_wait_for, - max_tries=default_max_tries): +def enable_configsvr( + config_data, wait_for=default_wait_for, max_tries=default_max_tries +): if config_data is None: juju_log("enable_configsvr: config_data not defined.") - return(False) + return False # Stop any running config servers disable_configsvr(config_data["config_server_port"]) # Make sure dbpath and logpath exist + subprocess.call(["mkdir", "-p", "%s" % config_data["config_server_dbpath"]]) subprocess.call( - [ - 'mkdir', - '-p', - '%s' % config_data['config_server_dbpath'] - ] - ) - subprocess.call( - [ - 'mkdir', - '-p', - '%s' % os.path.dirname(config_data['config_server_logpath']) - ] + ["mkdir", "-p", "%s" % os.path.dirname(config_data["config_server_logpath"])] ) # Start the config server @@ -768,9 +746,9 @@ def enable_configsvr(config_data, wait_for=default_wait_for, cmd_line = "mongod" cmd_line += " --configsvr" cmd_line += " --bind_ip {}".format(choose_bind_ip(config_data["bind_ip"])) - cmd_line += " --port %d" % config_data['config_server_port'] - cmd_line += " --dbpath %s" % config_data['config_server_dbpath'] - cmd_line += " --logpath %s" % config_data['config_server_logpath'] + cmd_line += " --port %d" % config_data["config_server_port"] + cmd_line += " --dbpath %s" % config_data["config_server_dbpath"] + cmd_line += " --logpath %s" % config_data["config_server_logpath"] cmd_line += " --pidfilepath /var/run/mongodb/configsvr.pid" cmd_line += " --replSet {}".format(config_data["replicaset"]) cmd_line += " --fork" @@ -778,99 +756,101 @@ def enable_configsvr(config_data, wait_for=default_wait_for, retVal = configsvr_ready(wait_for, max_tries) if retVal: - open_port(config_data['config_server_port']) - if config_data['web_admin_ui']: - port = int(config_data['config_server_port']) + 1000 + open_port(config_data["config_server_port"]) + if config_data["web_admin_ui"]: + port = int(config_data["config_server_port"]) + 1000 open_port(port) juju_log("enable_configsvr returns: %s" % retVal) - return(retVal) + return retVal def mongos_status(wait_for=default_wait_for, max_tries=default_max_tries): config_data = config() current_try = 0 - while (process_check_pidfile('/var/run/mongodb/mongos.pid') != ( - None, None)) and not port_check( - unit_get('private-address'), - config_data['mongos_port']) and current_try < max_tries: + while ( + (process_check_pidfile("/var/run/mongodb/mongos.pid") != (None, None)) + and not port_check(unit_get("private-address"), config_data["mongos_port"]) + and current_try < max_tries + ): juju_log("mongos_status: Waiting for Mongo shell to be ready ...") time.sleep(wait_for) current_try += 1 - retVal = \ - (process_check_pidfile('/var/run/mongodb/mongos.pid') != ( - None, None)) == port_check( - unit_get('private-address'), - config_data['mongos_port']) is True + retVal = ( + (process_check_pidfile("/var/run/mongodb/mongos.pid") != (None, None)) + == port_check(unit_get("private-address"), config_data["mongos_port"]) + is True + ) if retVal: - return(process_check_pidfile('/var/run/mongodb/mongos.pid')) + return process_check_pidfile("/var/run/mongodb/mongos.pid") else: - return((None, None)) + return (None, None) def mongos_ready(wait_for=default_wait_for, max_tries=default_max_tries): - return(mongos_status(wait_for, max_tries) != (None, None)) + return mongos_status(wait_for, max_tries) != (None, None) def disable_mongos(port=None): if port is None: juju_log("disable_mongos: port not defined") - return(False) + return False try: - pid = open('/var/run/mongodb/mongos.pid').read() + pid = open("/var/run/mongodb/mongos.pid").read() os.kill(int(pid), signal.SIGTERM) - os.unlink('/var/run/mongodb/mongos.pid') + os.unlink("/var/run/mongodb/mongos.pid") retVal = True except Exception as e: - juju_log('no mongo router running ...') + juju_log("no mongo router running ...") juju_log("Exception: %s" % str(e)) retVal = False finally: juju_log("disable_mongos returns %s" % retVal) close_port(port) - return(retVal) + return retVal -def enable_mongos(config_data=None, config_servers=None, replicaset=None, - wait_for=default_wait_for, max_tries=default_max_tries): +def enable_mongos( + config_data=None, + config_servers=None, + replicaset=None, + wait_for=default_wait_for, + max_tries=default_max_tries, +): juju_log("enable_mongos") if config_data is None or config_servers is None: juju_log("enable_mongos: config_data and config_servers are mandatory") - return(False) + return False if not isinstance(config_servers, list): juju_log("enable_mongos: config_servers must be a list") - return(False) + return False if len(config_servers) < 3: # MongoDB 3.2 deprecates the use of three mirrored mongod instances # for config servers. juju_log("enable_mongos: Not enough config servers yet...") - return(True) + return True disable_mongos() # Make sure logpath exist subprocess.call( - [ - 'mkdir', - '-p', - '%s' % os.path.dirname(config_data['mongos_logpath']) - ] + ["mkdir", "-p", "%s" % os.path.dirname(config_data["mongos_logpath"])] ) cmd_line = "mongos" - cmd_line += " --logpath %s" % config_data['mongos_logpath'] + cmd_line += " --logpath %s" % config_data["mongos_logpath"] cmd_line += " --pidfilepath /var/run/mongodb/mongos.pid" - cmd_line += " --port %d" % config_data['mongos_port'] + cmd_line += " --port %d" % config_data["mongos_port"] cmd_line += " --fork" # Note(aluria): --configdb used to have a list of comma-separated # server:port values. Such list now needs to be prepended by # repSetConfigName/ - cmd_line += ' --configdb {}/{}'.format(replicaset, ','.join(config_servers[0:3])) + cmd_line += " --configdb {}/{}".format(replicaset, ",".join(config_servers[0:3])) juju_log("enable_mongos: cmd_line: %s" % cmd_line) subprocess.call(cmd_line, shell=True) retVal = mongos_ready(wait_for, max_tries) if retVal: - open_port(config_data['mongos_port']) + open_port(config_data["mongos_port"]) juju_log("enable_mongos returns: %s" % retVal) - return(retVal) + return retVal def load_config_servers(mongos_list=None): @@ -878,42 +858,45 @@ def load_config_servers(mongos_list=None): retVal = [line.strip() for line in open(mongos_list).readlines()] else: retVal = [] - return(retVal) + return retVal def restart_mongod(wait_for=default_wait_for, max_tries=default_max_tries): - my_hostname = unit_get('private-address') - my_port = config('port') + my_hostname = unit_get("private-address") + my_port = config("port") current_try = 0 - service('stop', 'mongodb') - if os.path.exists('/var/lib/mongodb/mongod.lock'): - os.remove('/var/lib/mongodb/mongod.lock') + service("stop", "mongodb") + if os.path.exists("/var/lib/mongodb/mongod.lock"): + os.remove("/var/lib/mongodb/mongod.lock") - if not service('start', 'mongodb'): + if not service("start", "mongodb"): return False - while (service('status', 'mongodb') and not port_check( - my_hostname, my_port) and current_try < max_tries): + while ( + service("status", "mongodb") + and not port_check(my_hostname, my_port) + and current_try < max_tries + ): juju_log( "restart_mongod: Waiting for MongoDB to be ready ({}/{})".format( - current_try, max_tries)) + current_try, max_tries + ) + ) time.sleep(wait_for) current_try += 1 - return( - (service('status', 'mongodb') == port_check(my_hostname, - my_port)) is True) + return (service("status", "mongodb") == port_check(my_hostname, my_port)) is True def backup_cronjob(disable=False): """Generate the cronjob to backup with mongodump.""" - juju_log('Setting up cronjob') + juju_log("Setting up cronjob") config_data = config() - backupdir = config_data['backup_directory'] - bind_ip = config_data['bind_ip'] - cron_file = '/etc/cron.d/mongodb' - cron_runtime = '@daily' + backupdir = config_data["backup_directory"] + bind_ip = config_data["bind_ip"] + cron_file = "/etc/cron.d/mongodb" + cron_runtime = "@daily" # Disable or not remove it and regenerate it with new config data. if exists(cron_file): @@ -921,137 +904,143 @@ def backup_cronjob(disable=False): if not disable: tpl_data = { - 'backup_copies': config_data['backup_copies_kept'], - 'backup_directory': backupdir, - 'unique_name': os.environ.get('JUJU_UNIT_NAME', bind_ip), - 'bind_ip': bind_ip, - 'port': config_data['port'], + "backup_copies": config_data["backup_copies_kept"], + "backup_directory": backupdir, + "unique_name": os.environ.get("JUJU_UNIT_NAME", bind_ip), + "bind_ip": bind_ip, + "port": config_data["port"], } - script_filename = '/var/lib/mongodb/cronbackup.py' - script_template = 'templates/backup.py.tpl' + script_filename = "/var/lib/mongodb/cronbackup.py" + script_template = "templates/backup.py.tpl" - juju_log('Writing out cronbackup.py') + juju_log("Writing out cronbackup.py") with open(script_template) as handle: template = Template(handle.read()) rendered = template.substitute(tpl_data) - with open(script_filename, 'w') as output: + with open(script_filename, "w") as output: output.writelines(rendered) chmod(script_filename, 0o755) - juju_log('Installing cron.d/mongodb') + juju_log("Installing cron.d/mongodb") if exists(cron_file): remove(cron_file) - with open(cron_file, 'w') as output: - output.write(""" + with open(cron_file, "w") as output: + output.write( + """ SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin %s ubuntu python %s -""" % (cron_runtime, script_filename)) +""" + % (cron_runtime, script_filename) + ) ############################################################################### # Hook functions ############################################################################### -@hooks.hook('install.real') +@hooks.hook("install.real") def install_hook(): - juju_log('Begin install hook.') + juju_log("Begin install hook.") execd_preinstall() - status_set('maintenance', 'Installing packages') + status_set("maintenance", "Installing packages") juju_log("Installing mongodb") - add_source(config('source'), config('key')) + add_source(config("source"), config("key")) apt_update(fatal=True) apt_install(packages=INSTALL_PACKAGES, fatal=True) -@hooks.hook('config-changed') +@hooks.hook("config-changed") def config_changed(): # noqa: C901 juju_log("Entering config_changed") - status_set('maintenance', 'Configuring unit') + status_set("maintenance", "Configuring unit") config_data = config() juju_log("config_data: {}".format(config_data), level=DEBUG) mongodb_config_hash = file_hash(default_mongodb_config) mongodb_config = open(default_mongodb_config).read() - if config_data.changed('volume-ephemeral-storage') or \ - config_data.changed('volume-map') or \ - config_data.changed('volume-dev-regexp'): + if ( + config_data.changed("volume-ephemeral-storage") + or config_data.changed("volume-map") + or config_data.changed("volume-dev-regexp") + ): config_changed_volume() # current ports - current_mongodb_port = re.search(r'^#*port\s+=\s+(\w+)', - mongodb_config, - re.MULTILINE).group(1) + current_mongodb_port = re.search( + r"^#*port\s+=\s+(\w+)", mongodb_config, re.MULTILINE + ).group(1) current_web_admin_ui_port = int(current_mongodb_port) + 1000 - new_web_admin_ui_port = int(config_data['port']) + 1000 + new_web_admin_ui_port = int(config_data["port"]) + 1000 - juju_log("Configured mongodb port: {}".format(current_mongodb_port), - level=DEBUG) - public_address = unit_get('public-address') + juju_log("Configured mongodb port: {}".format(current_mongodb_port), level=DEBUG) + public_address = unit_get("public-address") juju_log("unit's public_address: {}".format(public_address), level=DEBUG) - private_address = unit_get('private-address') + private_address = unit_get("private-address") juju_log("unit's private_address: {}".format(private_address), level=DEBUG) # Update mongodb configuration file mongodb_config = mongodb_conf(config_data) update_file(default_mongodb_config, mongodb_config) - if config_data['backups_enabled']: + if config_data["backups_enabled"]: backup_cronjob() else: backup_cronjob(disable=True) # web_admin_ui - if config_data['web_admin_ui']: + if config_data["web_admin_ui"]: open_port(new_web_admin_ui_port) else: close_port(current_web_admin_ui_port) # replicaset_master - if config_data['replicaset_master'] != "auto": - enable_replset(config_data['replicaset']) - join_replset(config_data['replicaset_master']) + if config_data["replicaset_master"] != "auto": + enable_replset(config_data["replicaset"]) + join_replset(config_data["replicaset_master"]) # extra demon options - update_daemon_options(config_data['extra_daemon_options']) + update_daemon_options(config_data["extra_daemon_options"]) # write mongodb logrotate configuration file write_logrotate_config(config_data) # restart mongodb if the configuration file has changed. if file_hash(default_mongodb_config) != mongodb_config_hash: - status_set('maintenance', 'Restarting mongod') + status_set("maintenance", "Restarting mongod") restart_mongod() # attach to replSet ( if needed ) - if config_data['replicaset_master'] != "auto": - join_replset(config_data['replicaset_master'], private_address) + if config_data["replicaset_master"] != "auto": + join_replset(config_data["replicaset_master"], private_address) # arbiter - if config_data['replicaset_master'] != 'auto': - if config_data['arbiter'] != "disabled" and\ - config_data['replicaset_master'] != "auto": - if config_data['arbiter'] == 'enable': - enable_arbiter(config_data['replicaset_master'], - "%s:%s" % (private_address, - config_data['port'])) + if config_data["replicaset_master"] != "auto": + if ( + config_data["arbiter"] != "disabled" + and config_data["replicaset_master"] != "auto" + ): + if config_data["arbiter"] == "enable": + enable_arbiter( + config_data["replicaset_master"], + "%s:%s" % (private_address, config_data["port"]), + ) else: - enable_arbiter(config_data['replicaset_master'], - config_data['arbiter']) + enable_arbiter(config_data["replicaset_master"], config_data["arbiter"]) # expose necessary ports - update_service_ports([current_mongodb_port], [config_data['port']]) + update_service_ports([current_mongodb_port], [config_data["port"]]) - if config_data['web_admin_ui']: + if config_data["web_admin_ui"]: current_web_admin_ui_port = int(current_mongodb_port) + 1000 - new_web_admin_ui_port = int(config_data['port']) + 1000 + new_web_admin_ui_port = int(config_data["port"]) + 1000 close_port(current_web_admin_ui_port) open_port(new_web_admin_ui_port) @@ -1065,12 +1054,11 @@ def config_changed(): # noqa: C901 juju_log("config_changed: Exception: %s" % str(e)) if configsvr_pid is not None: - configsvr_port = re.search(r'--port (\w+)', - configsvr_cmd_line).group(2) + configsvr_port = re.search(r"--port (\w+)", configsvr_cmd_line).group(2) disable_configsvr(configsvr_port) - enable_configsvr(config_data['config_server_port']) + enable_configsvr(config_data["config_server_port"]) else: - open_port(config_data['config_server_port']) + open_port(config_data["config_server_port"]) # update mongos information and port try: @@ -1082,90 +1070,93 @@ def config_changed(): # noqa: C901 juju_log("config_changed: Exceptions: %s" % str(e)) if mongos_pid is not None: - mongos_port = re.search(r'--port (\w+)', mongos_cmd_line).group(1) + mongos_port = re.search(r"--port (\w+)", mongos_cmd_line).group(1) disable_mongos(mongos_port) - enable_mongos(config_data['mongos_port']) + enable_mongos(config_data["mongos_port"]) else: - open_port(config_data['mongos_port']) + open_port(config_data["mongos_port"]) update_nrpe_config() application_version_set(get_mongod_version()) update_status() juju_log("About to leave config_changed", level=DEBUG) - return(True) + return True -@hooks.hook('start') +@hooks.hook("start") def start_hook(): juju_log("start_hook") retVal = restart_mongod() juju_log("start_hook returns: %s" % retVal) - return(retVal) + return retVal -@hooks.hook('stop') +@hooks.hook("stop") def stop_hook(): juju_log("stop_hook") try: - retVal = service('stop', 'mongodb') - os.remove('/var/lib/mongodb/mongod.lock') + retVal = service("stop", "mongodb") + os.remove("/var/lib/mongodb/mongod.lock") # FIXME Need to check if this is still needed except Exception as e: juju_log(str(e)) retVal = False finally: juju_log("stop_hook returns: %s" % retVal) - return(retVal) + return retVal -@hooks.hook('database-relation-joined') +@hooks.hook("database-relation-joined") def database_relation_joined(): juju_log("database_relation_joined") - my_hostname = unit_get('private-address') - my_port = config('port') - my_replset = config('replicaset') + my_hostname = unit_get("private-address") + my_port = config("port") + my_replset = config("replicaset") juju_log("my_hostname: %s" % my_hostname) juju_log("my_port: %s" % my_port) juju_log("my_replset: %s" % my_replset) - relation_data = {'hostname': my_hostname, - 'port': my_port, - 'type': 'database', - 'version': get_mongod_version(), - } + relation_data = { + "hostname": my_hostname, + "port": my_port, + "type": "database", + "version": get_mongod_version(), + } - if len(peer_units('replica-set')) > 1: - relation_data['replset'] = my_replset + if len(peer_units("replica-set")) > 1: + relation_data["replset"] = my_replset relation_set(relation_id(), relation_data) -@hooks.hook('replica-set-relation-joined') +@hooks.hook("replica-set-relation-joined") def replica_set_relation_joined(): juju_log("replica_set_relation_joined-start") - my_hostname = unit_get('private-address') - my_port = config('port') - my_replset = config('replicaset') + my_hostname = unit_get("private-address") + my_port = config("port") + my_replset = config("replicaset") - my_install_order = os.environ['JUJU_UNIT_NAME'].split('/')[1] + my_install_order = os.environ["JUJU_UNIT_NAME"].split("/")[1] juju_log("my_hostname: %s" % my_hostname) juju_log("my_port: %s" % my_port) juju_log("my_replset: %s" % my_replset) juju_log("my_install_order: %s" % my_install_order) # do not restart mongodb if config has not changed if enable_replset(my_replset): - juju_log('Restarting mongodb after config change (enable replset)', - level=DEBUG) - status_set('maintenance', 'Restarting mongod to enable replicaset') + juju_log("Restarting mongodb after config change (enable replset)", level=DEBUG) + status_set("maintenance", "Restarting mongod to enable replicaset") restart_mongod() - relation_set(relation_id(), { - 'hostname': my_hostname, - 'port': my_port, - 'replset': my_replset, - 'install-order': my_install_order, - 'type': 'replset', - }) + relation_set( + relation_id(), + { + "hostname": my_hostname, + "port": my_port, + "replset": my_replset, + "install-order": my_install_order, + "type": "replset", + }, + ) update_status() juju_log("replica_set_relation_joined-finish") @@ -1173,24 +1164,29 @@ def replica_set_relation_joined(): def rs_add(host): if not is_bionic_or_greater(): - return mongo_client_smart('localhost', 'rs.add("%s")' % host) + return mongo_client_smart("localhost", 'rs.add("%s")' % host) command = 'rs.add("%s")' % host if host is None: raise ValueError("missing host") else: - cmd_line = ['mongo', '--quiet', '--host', "localhost", - '--eval', 'printjson(%s)' % command] + cmd_line = [ + "mongo", + "--quiet", + "--host", + "localhost", + "--eval", + "printjson(%s)" % command, + ] juju_log("Executing: %s" % cmd_line, level=DEBUG) run(cmd_line) for i in range(MONGO_CLIENT_RETRIES): - c = MongoClient('localhost') + c = MongoClient("localhost") subprocess.check_output(cmd_line) - r = run_admin_command(c, 'replSetGetStatus') + r = run_admin_command(c, "replSetGetStatus") members = r["members"] - ok = [m for m in members - if m['name'] == host and m['state'] == MONGO_SECONDARY] + ok = [m for m in members if m["name"] == host and m["state"] == MONGO_SECONDARY] if ok: return ok @@ -1200,26 +1196,27 @@ def rs_add(host): def am_i_primary(): - c = MongoClient('localhost') + c = MongoClient("localhost") for i in range(10): try: - r = run_admin_command(c, 'replSetGetStatus') + r = run_admin_command(c, "replSetGetStatus") pretty_r = pprint.pformat(r) - juju_log('am_i_primary: replSetGetStatus returned: %s' % pretty_r, - level=DEBUG) - return r['myState'] == MONGO_PRIMARY + juju_log( + "am_i_primary: replSetGetStatus returned: %s" % pretty_r, level=DEBUG + ) + return r["myState"] == MONGO_PRIMARY except OperationFailure as e: - juju_log('am_i_primary: OperationError: %s' % str(e), level=DEBUG) - if 'replSetInitiate - should come online shortly' in str(e): + juju_log("am_i_primary: OperationError: %s" % str(e), level=DEBUG) + if "replSetInitiate - should come online shortly" in str(e): # replSet initialization in progress continue - elif 'EMPTYCONFIG' in str(e): + elif "EMPTYCONFIG" in str(e): # replication not initialized yet return False - elif 'no replset config has been received' in str(e): + elif "no replset config has been received" in str(e): # replication not initialized yet (Mongo3.4+) return False - elif 'not running with --replSet' in str(e): + elif "not running with --replSet" in str(e): # replicaset not configured return False else: @@ -1228,7 +1225,7 @@ def am_i_primary(): time.sleep(1.5) # Raise an error if we exhausted the maximum amount of trials - raise TimeoutException('Unable to determine if local unit is primary') + raise TimeoutException("Unable to determine if local unit is primary") def get_replicaset_status(): @@ -1241,24 +1238,26 @@ def get_replicaset_status(): or can be the string of an exception while getting the status """ - c = MongoClient('localhost') + c = MongoClient("localhost") try: - r = run_admin_command(c, 'replSetGetStatus') - for member in r['members']: - if 'self' in member: - return member['stateStr'] + r = run_admin_command(c, "replSetGetStatus") + for member in r["members"]: + if "self" in member: + return member["stateStr"] # if 'self' was not found in the output, then log a warning and print # the output given by replSetGetStatus r_pretty = pprint.pformat(r) - juju_log('get_replicaset_status() failed to get replicaset state: ' - '%s' % r_pretty, level=WARNING) - return 'Unknown replica set state' + juju_log( + "get_replicaset_status() failed to get replicaset state: %s" % r_pretty, + level=WARNING, + ) + return "Unknown replica set state" except OperationFailure as e: - juju_log('get_replicaset_status() exception: %s' % str(e), DEBUG) - if 'not running with --replSet' in str(e): - return 'not in replicaset' + juju_log("get_replicaset_status() exception: %s" % str(e), DEBUG) + if "not running with --replSet" in str(e): + return "not in replicaset" else: return str(e) @@ -1268,8 +1267,8 @@ def get_mongod_version(): Mainly used for application_set_version in config-changed hook """ - c = MongoClient('localhost', serverSelectionTimeoutMS=60000) - return c.server_info()['version'] + c = MongoClient("localhost", serverSelectionTimeoutMS=60000) + return c.server_info()["version"] # Retry until the replica set is in active state, @@ -1279,301 +1278,302 @@ def get_mongod_version(): @retry_on_exception(num_retries=45, base_delay=1) def wait_until_replset_is_active(): status = update_status() - if status != 'active': - raise Exception('ReplicaSet not active: {}'.format(status)) + if status != "active": + raise Exception("ReplicaSet not active: {}".format(status)) -@hooks.hook('replica-set-relation-changed') +@hooks.hook("replica-set-relation-changed") def replica_set_relation_changed(): - private_address = unit_get('private-address') - remote_hostname = relation_get('hostname') + private_address = unit_get("private-address") + remote_hostname = relation_get("hostname") - juju_log('replica_set_relation_changed-start') - juju_log('local unit: %s, joining_unit: %s' % (private_address, - remote_hostname), - level=DEBUG) + juju_log("replica_set_relation_changed-start") + juju_log( + "local unit: %s, joining_unit: %s" % (private_address, remote_hostname), + level=DEBUG, + ) if remote_hostname is None: - juju_log('Joiner not ready yet... bailing out') + juju_log("Joiner not ready yet... bailing out") return # Initialize the replicaset - we do this only on the leader! if is_leader(): - juju_log('Initializing replicaset') - status_set('maintenance', 'Initializing replicaset') + juju_log("Initializing replicaset") + status_set("maintenance", "Initializing replicaset") init_replset() - unit = "%s:%s" % (private_address, config('port')) - unit_remote = "%s:%s" % (remote_hostname, relation_get('port')) + unit = "%s:%s" % (private_address, config("port")) + unit_remote = "%s:%s" % (remote_hostname, relation_get("port")) # If this is primary, add joined unit to replicaset if am_i_primary(): - juju_log('Adding new secondary... %s' % unit_remote, level=DEBUG) + juju_log("Adding new secondary... %s" % unit_remote, level=DEBUG) join_replset(unit, unit_remote) wait_until_replset_is_active() - juju_log('replica_set_relation_changed-finish') + juju_log("replica_set_relation_changed-finish") -@hooks.hook('replica-set-relation-departed') +@hooks.hook("replica-set-relation-departed") def replica_set_relation_departed(): - juju_log('replica_set_relation_departed-start') + juju_log("replica_set_relation_departed-start") if not am_i_primary(): - juju_log('replica_set_relation_departed-finish') + juju_log("replica_set_relation_departed-finish") return - unit_address, unit_port = unit_get('private-address'), config('port') - remote_address = relation_get('private-address') - remote_port = relation_get('port') + unit_address, unit_port = unit_get("private-address"), config("port") + remote_address = relation_get("private-address") + remote_port = relation_get("port") # If I am the unit being removed, step me down from being primary if (unit_address, unit_port) == (remote_address, remote_port): - juju_log('Stepping down from being primary...') + juju_log("Stepping down from being primary...") global was_i_primary was_i_primary = True - mongo_client('localhost', 'rs.stepDown()') - juju_log('replica_set_relation_departed-finish') + mongo_client("localhost", "rs.stepDown()") + juju_log("replica_set_relation_departed-finish") return - unit = "%s:%s" % (unit_get('private-address'), - config('port')) - unit_remote = "%s:%s" % (relation_get('hostname'), - relation_get('port')) + unit = "%s:%s" % (unit_get("private-address"), config("port")) + unit_remote = "%s:%s" % (relation_get("hostname"), relation_get("port")) leave_replset(unit, unit_remote) - juju_log('Removed %s from replicaset' % unit_remote) - juju_log('replica_set_relation_departed-finish') + juju_log("Removed %s from replicaset" % unit_remote) + juju_log("replica_set_relation_departed-finish") -@hooks.hook('replica-set-relation-broken') +@hooks.hook("replica-set-relation-broken") def replica_set_relation_broken(): - juju_log('replica_set_relation_broken-start') + juju_log("replica_set_relation_broken-start") if am_i_primary(): - juju_log('I was primary - removing myself via new primary.', 'DEBUG') - mongo_client('localhost', 'rs.stepDown()') - time.sleep(15) # give some time to for re-election to happen + juju_log("I was primary - removing myself via new primary.", "DEBUG") + mongo_client("localhost", "rs.stepDown()") + time.sleep(15) # give some time to for re-election to happen - c = MongoClient('localhost') - r = c.admin.command('isMaster') + c = MongoClient("localhost") + r = c.admin.command("isMaster") try: - master_node = r['primary'] + master_node = r["primary"] except KeyError: pass - if 'master_node' in locals(): # unit is part of replicaset, remove it! - unit = "%s:%s" % (unit_get('private-address'), config('port')) - juju_log('Removing myself via %s' % (master_node), 'DEBUG') + if "master_node" in locals(): # unit is part of replicaset, remove it! + unit = "%s:%s" % (unit_get("private-address"), config("port")) + juju_log("Removing myself via %s" % (master_node), "DEBUG") leave_replset(master_node, unit) - juju_log('replica_set_relation_broken-finish') + juju_log("replica_set_relation_broken-finish") -@hooks.hook('data-relation-joined') +@hooks.hook("data-relation-joined") def data_relation_joined(): juju_log("data_relation_joined") - return relation_set(relation_id(), - {'mountpoint': '/srv/juju/mongodb-data'}) + return relation_set(relation_id(), {"mountpoint": "/srv/juju/mongodb-data"}) -@hooks.hook('data-relation-changed') +@hooks.hook("data-relation-changed") def data_relation_changed(): juju_log("data_relation_changed") if volume_get_id_for_storage_subordinate() is None: juju_log("mountpoint from storage subordinate not ready, let's wait") - return(True) + return True config_changed_volume() -@hooks.hook('data-relation-departed') +@hooks.hook("data-relation-departed") def data_relation_departed(): juju_log("data_relation_departed") - return(config_changed_volume()) + return config_changed_volume() -@hooks.hook('configsvr-relation-joined') +@hooks.hook("configsvr-relation-joined") def configsvr_relation_joined(): juju_log("configsvr_relation_joined") - my_hostname = unit_get('private-address') - my_port = config('config_server_port') - my_install_order = os.environ['JUJU_UNIT_NAME'].split('/')[1] - my_replicaset = config('replicaset') - relation_set(relation_id(), { - 'hostname': my_hostname, - 'port': my_port, - 'install-order': my_install_order, - 'replset': my_replicaset, - 'type': 'configsvr', - }) - - -@hooks.hook('configsvr-relation-changed') + my_hostname = unit_get("private-address") + my_port = config("config_server_port") + my_install_order = os.environ["JUJU_UNIT_NAME"].split("/")[1] + my_replicaset = config("replicaset") + relation_set( + relation_id(), + { + "hostname": my_hostname, + "port": my_port, + "install-order": my_install_order, + "replset": my_replicaset, + "type": "configsvr", + }, + ) + + +@hooks.hook("configsvr-relation-changed") def configsvr_relation_changed(): juju_log("configsvr_relation_changed") config_data = config() - my_port = config_data['config_server_port'] + my_port = config_data["config_server_port"] disable_configsvr(my_port) retVal = enable_configsvr(config_data) juju_log("configsvr_relation_changed returns: %s" % retVal) - return(retVal) + return retVal -@hooks.hook('mongos-cfg-relation-joined') -@hooks.hook('mongos-relation-joined') +@hooks.hook("mongos-cfg-relation-joined") +@hooks.hook("mongos-relation-joined") def mongos_relation_joined(): juju_log("mongos_relation_joined") - my_hostname = unit_get('private-address') - my_port = config('mongos_port') - my_install_order = os.environ['JUJU_UNIT_NAME'].split('/')[1] - my_replicaset = config('replicaset') - relation_set(relation_id(), { - 'hostname': my_hostname, - 'port': my_port, - 'install-order': my_install_order, - 'replset': my_replicaset, - 'type': 'mongos' - }) - - -@hooks.hook('mongos-cfg-relation-changed') -@hooks.hook('mongos-relation-changed') + my_hostname = unit_get("private-address") + my_port = config("mongos_port") + my_install_order = os.environ["JUJU_UNIT_NAME"].split("/")[1] + my_replicaset = config("replicaset") + relation_set( + relation_id(), + { + "hostname": my_hostname, + "port": my_port, + "install-order": my_install_order, + "replset": my_replicaset, + "type": "mongos", + }, + ) + + +@hooks.hook("mongos-cfg-relation-changed") +@hooks.hook("mongos-relation-changed") def mongos_relation_changed(): juju_log("mongos_relation_changed") config_data = config() retVal = False - hostname = relation_get('hostname') - port = relation_get('port') - replicaset = relation_get('replset') - rel_type = relation_get('type') + hostname = relation_get("hostname") + port = relation_get("port") + replicaset = relation_get("replset") + rel_type = relation_get("type") if hostname is None or port is None or rel_type is None: - juju_log("mongos_relation_changed: relation data not ready.", - level=DEBUG) + juju_log("mongos_relation_changed: relation data not ready.", level=DEBUG) return - if rel_type == 'configsvr': + if rel_type == "configsvr": config_servers = load_config_servers(default_mongos_list) juju_log("Adding config server: %s:%s" % (hostname, port), level=DEBUG) if hostname and port and "{}:{}".format(hostname, port) not in config_servers: config_servers.append("%s:%s" % (hostname, port)) - disable_mongos(config_data['mongos_port']) + disable_mongos(config_data["mongos_port"]) retVal = enable_mongos(config_data, config_servers, replicaset) if retVal: - update_file(default_mongos_list, '\n'.join(config_servers)) - elif rel_type == 'database': + update_file(default_mongos_list, "\n".join(config_servers)) + elif rel_type == "database": if mongos_ready(): - mongos_host = "%s:%s" % ( - unit_get('private-address'), - config('mongos_port')) - shard_command1 = "sh.addShard(\"%s:%s\")" % (hostname, port) + mongos_host = "%s:%s" % (unit_get("private-address"), config("mongos_port")) + shard_command1 = 'sh.addShard("%s:%s")' % (hostname, port) mongo_client(mongos_host, shard_command1) - replicaset = relation_get('replset') - shard_command2 = "sh.addShard(\"%s/%s:%s\")" % \ - (replicaset, hostname, port) + replicaset = relation_get("replset") + shard_command2 = 'sh.addShard("%s/%s:%s")' % (replicaset, hostname, port) mongo_client(mongos_host, shard_command2) else: - juju_log("mongos_relation_change: undefined rel_type: %s" % rel_type, - level=DEBUG) + juju_log( + "mongos_relation_change: undefined rel_type: %s" % rel_type, level=DEBUG + ) return juju_log("mongos_relation_changed returns: %s" % retVal, level=DEBUG) -@hooks.hook('mongos-relation-broken') +@hooks.hook("mongos-relation-broken") def mongos_relation_broken(): config_servers = load_config_servers(default_mongos_list) - for member in relations_of_type('mongos'): - hostname = relation_get('hostname', member) - port = relation_get('port', member) - if '%s:%s' % (hostname, port) in config_servers: - config_servers.remove('%s:%s' % (hostname, port)) + for member in relations_of_type("mongos"): + hostname = relation_get("hostname", member) + port = relation_get("port", member) + if "%s:%s" % (hostname, port) in config_servers: + config_servers.remove("%s:%s" % (hostname, port)) - update_file(default_mongos_list, '\n'.join(config_servers)) + update_file(default_mongos_list, "\n".join(config_servers)) -@hooks.hook('nrpe-external-master-relation-joined', - 'nrpe-external-master-relation-changed') +@hooks.hook( + "nrpe-external-master-relation-joined", "nrpe-external-master-relation-changed" +) def update_nrpe_config(): # Find out if nrpe set nagios_hostname hostname = None host_context = None - for rel in relations_of_type('nrpe-external-master'): - if 'nagios_hostname' in rel: - hostname = rel['nagios_hostname'] - host_context = rel['nagios_host_context'] + for rel in relations_of_type("nrpe-external-master"): + if "nagios_hostname" in rel: + hostname = rel["nagios_hostname"] + host_context = rel["nagios_host_context"] break nrpe = NRPE(hostname=hostname) - apt_install('python3-dbus') + apt_install("python3-dbus") if host_context: current_unit = "%s:%s" % (host_context, local_unit()) else: current_unit = local_unit() - if lsb_release()['DISTRIB_RELEASE'] > '15.04': - check_mongo_script = 'check_systemd.py mongodb' + if lsb_release()["DISTRIB_RELEASE"] > "15.04": + check_mongo_script = "check_systemd.py mongodb" else: - check_mongo_script = 'check_upstart_job mongodb' + check_mongo_script = "check_upstart_job mongodb" nrpe.add_check( - shortname='mongodb', - description='process check {%s}' % current_unit, + shortname="mongodb", + description="process check {%s}" % current_unit, check_cmd=check_mongo_script, ) nrpe.write() -@hooks.hook('upgrade-charm') +@hooks.hook("upgrade-charm") def uprade_charm(): - juju_log('upgrade-charm: removing --replset from upstart script') + juju_log("upgrade-charm: removing --replset from upstart script") remove_replset_from_upstart() - juju_log('upgrade-charm: removing --rest from upstart script') + juju_log("upgrade-charm: removing --rest from upstart script") remove_rest_from_upstart() -@hooks.hook('update-status') +@hooks.hook("update-status") def update_status(): """ Returns: workload_state (so that some hooks know they need to re-run update_status if needed) """ - workload = 'active' - status = 'Unit is ready' + workload = "active" + status = "Unit is ready" - if is_relation_made('replica-set'): + if is_relation_made("replica-set"): # only check for replica-set state if the relation was made which means # more than 1 units were deployed and peer related. mongo_status = get_replicaset_status() - if mongo_status in ('PRIMARY', 'SECONDARY'): - workload = 'active' - status = 'Unit is ready as ' + mongo_status - elif mongo_status in ('not in replicaset',): - workload = 'active' - status = 'Unit is ready, ' + mongo_status + if mongo_status in ("PRIMARY", "SECONDARY"): + workload = "active" + status = "Unit is ready as " + mongo_status + elif mongo_status in ("not in replicaset"): + workload = "active" + status = "Unit is ready, " + mongo_status else: - workload = 'maintenance' + workload = "maintenance" status = mongo_status - juju_log('mongo_status is unknown: {}'.format(status), level=DEBUG) + juju_log("mongo_status is unknown: {}".format(status), level=DEBUG) - juju_log('Setting workload: {} - {}'.format(workload, status), level=DEBUG) + juju_log("Setting workload: {} - {}".format(workload, status), level=DEBUG) status_set(workload, status) return workload def run(command, exit_on_error=True): - '''Run a command and return the output.''' + """Run a command and return the output.""" try: juju_log(command) - return subprocess.check_output( - command, stderr=subprocess.STDOUT, shell=True) + return subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: juju_log("status=%d, output=%s" % (e.returncode, e.output)) if exit_on_error: @@ -1594,9 +1594,9 @@ def volume_get_volid_from_volume_map(): config_data = config() volume_map = {} try: - volume_map = yaml.load(config_data['volume-map'].strip()) + volume_map = yaml.load(config_data["volume-map"].strip()) if volume_map: - juju_unit_name = os.environ['JUJU_UNIT_NAME'] + juju_unit_name = os.environ["JUJU_UNIT_NAME"] volid = volume_map.get(juju_unit_name) juju_log("Juju unit name: %s Volid:%s" % (juju_unit_name, volid)) return volid @@ -1631,15 +1631,13 @@ def volume_mount_point_from_volid(volid): def volume_get_id_for_storage_subordinate(): # storage charm is a subordinate so we should only ever have one # relation_id for the data relation - ids = relation_ids('data') + ids = relation_ids("data") if len(ids) > 0: - mountpoint = relation_get('mountpoint', - os.environ['JUJU_UNIT_NAME'], - ids[0]) + mountpoint = relation_get("mountpoint", os.environ["JUJU_UNIT_NAME"], ids[0]) - juju_log('mountpoint: %s' % (mountpoint,)) + juju_log("mountpoint: %s" % (mountpoint)) if mountpoint and os.path.exists(mountpoint): - return mountpoint.split('/')[-1] + return mountpoint.split("/")[-1] # Do we have a valid storage state? @@ -1652,14 +1650,15 @@ def volume_get_volume_id(): if volid: return volid - ephemeral_storage = config_data['volume-ephemeral-storage'] + ephemeral_storage = config_data["volume-ephemeral-storage"] volid = volume_get_volid_from_volume_map() - juju_unit_name = os.environ['JUJU_UNIT_NAME'] - if ephemeral_storage in [True, 'yes', 'Yes', 'true', 'True']: + juju_unit_name = os.environ["JUJU_UNIT_NAME"] + if ephemeral_storage in [True, "yes", "Yes", "true", "True"]: if volid: juju_log( "volume-ephemeral-storage is True, but" - "volume-map[{!r}] -> {}".format(juju_unit_name, volid)) + "volume-map[{!r}] -> {}".format(juju_unit_name, volid) + ) return None else: return "--ephemeral" @@ -1667,8 +1666,8 @@ def volume_get_volume_id(): if not volid: juju_log( "volume-ephemeral-storage is False, but " - "no volid found for volume-map[{!r}]".format( - juju_unit_name)) + "no volid found for volume-map[{!r}]".format(juju_unit_name) + ) return None juju_log("Volid:%s" % (volid)) return volid @@ -1687,7 +1686,8 @@ def config_changed_volume(): juju_log( "Disabled and stopped mongodb service, " "because of broken volume configuration - check " - "'volume-ephemeral-storage' and 'volume-map'") + "'volume-ephemeral-storage' and 'volume-map'" + ) sys.exit(1) if volume_is_permanent(volid): # config_changed_volume_apply will stop the service if it finds @@ -1701,7 +1701,8 @@ def config_changed_volume(): juju_log("current mounted volumes: {}".format(mounts)) juju_log( "Disabled and stopped mongodb service " - "(config_changed_volume_apply failure)") + "(config_changed_volume_apply failure)" + ) sys.exit(1) @@ -1709,14 +1710,13 @@ def config_changed_volume(): # shell helper def volume_init_and_mount(volid): juju_log("Initialize and mount volume") - command = ("scripts/volume-common.sh call " - "volume_init_and_mount %s" % volid) + command = "scripts/volume-common.sh call volume_init_and_mount %s" % volid run(command) return True def volume_get_all_mounted(): - command = ("mount |egrep /srv/juju") + command = "mount |egrep /srv/juju" status, output = subprocess.getstatusoutput(command) if status != 0: return None @@ -1736,20 +1736,20 @@ def volume_get_all_mounted(): def config_changed_volume_apply(): # noqa: C901 is too complex (12) config_data = config() data_directory_path = config_data["dbpath"] - assert(data_directory_path) + assert data_directory_path volid = volume_get_volume_id() if volid: volid_from_subordinate = volume_get_id_for_storage_subordinate() if volume_is_permanent(volid) and not volid_from_subordinate: if not volume_init_and_mount(volid): - juju_log( - "volume_init_and_mount failed, not applying changes") + juju_log("volume_init_and_mount failed, not applying changes") return False if not os.path.exists(data_directory_path): juju_log( "mongodb data dir {} not found, " - "not applying changes.".format(data_directory_path)) + "not applying changes.".format(data_directory_path) + ) return False mount_point = volume_mount_point_from_volid(volid) @@ -1757,18 +1757,21 @@ def config_changed_volume_apply(): # noqa: C901 is too complex (12) if not mount_point: juju_log( "invalid mount point from volid = {}, " - "not applying changes.".format(mount_point)) + "not applying changes.".format(mount_point) + ) return False if os.path.islink(data_directory_path): juju_log( "mongodb data dir '%s' already points " - "to %s, skipping storage changes." % (data_directory_path, - new_mongo_dir)) + "to %s, skipping storage changes." + % (data_directory_path, new_mongo_dir) + ) juju_log( "existing-symlink: to fix/avoid UID changes from " "previous units, doing: " - "chown -R mongodb:mongodb {}".format(new_mongo_dir)) + "chown -R mongodb:mongodb {}".format(new_mongo_dir) + ) run("chown -R mongodb:mongodb %s" % new_mongo_dir) return True @@ -1791,41 +1794,47 @@ def config_changed_volume_apply(): # noqa: C901 is too complex (12) juju_log("stop_hook() failed - can't migrate data.") return False if not os.path.exists(new_mongo_dir) or new_mongo_dir_just_created: - juju_log("migrating mongo data {}/ -> {}/".format( - data_directory_path, new_mongo_dir)) + juju_log( + "migrating mongo data {}/ -> {}/".format( + data_directory_path, new_mongo_dir + ) + ) # void copying PID file to perm storage (shouldn't be any...) - command = "rsync -a {}/ {}/".format( - data_directory_path, new_mongo_dir) + command = "rsync -a {}/ {}/".format(data_directory_path, new_mongo_dir) juju_log("run: {}".format(command)) run(command) try: - os.rename(data_directory_path, "{}-{}".format( - data_directory_path, int(time.time()))) - juju_log("NOTICE: symlinking {} -> {}".format( - new_mongo_dir, data_directory_path)) + os.rename( + data_directory_path, + "{}-{}".format(data_directory_path, int(time.time())), + ) + juju_log( + "NOTICE: symlinking {} -> {}".format(new_mongo_dir, data_directory_path) + ) os.symlink(new_mongo_dir, data_directory_path) juju_log( "after-symlink: to fix/avoid UID changes from " "previous units, doing: " - "chown -R mongodb:mongodb {}".format(new_mongo_dir)) + "chown -R mongodb:mongodb {}".format(new_mongo_dir) + ) run("chown -R mongodb:mongodb {}".format(new_mongo_dir)) return True except OSError: - juju_log("failed to symlink {} -> {}".format( - data_directory_path, mount_point)) + juju_log( + "failed to symlink {} -> {}".format(data_directory_path, mount_point) + ) return False else: - juju_log( - "Invalid volume storage configuration, not applying changes") + juju_log("Invalid volume storage configuration, not applying changes") return False # Write mongodb-server logrotate configuration -def write_logrotate_config(config_data, - conf_file='/etc/logrotate.d/mongodb-server'): +def write_logrotate_config(config_data, conf_file="/etc/logrotate.d/mongodb-server"): - juju_log('Writing {}.'.format(conf_file)) - contents = dedent(""" + juju_log("Writing {}.".format(conf_file)) + contents = dedent( + """ {logpath} {{ {logrotate-frequency} rotate {logrotate-rotate} @@ -1835,30 +1844,31 @@ def write_logrotate_config(config_data, compress notifempty missingok - }}""") + }}""" + ) contents = contents.format(**config_data) try: - with open(conf_file, 'w') as f: + with open(conf_file, "w") as f: f.write(contents) except IOError: - juju_log('Could not write {}.'.format(conf_file)) + juju_log("Could not write {}.".format(conf_file)) return False return True -@hooks.hook('pre-series-upgrade') +@hooks.hook("pre-series-upgrade") def pre_series_upgrade(): juju_log("Running prepare series upgrade hook", "INFO") if am_i_primary(): step_down_replset_primary() - service('stop', 'mongodb') + service("stop", "mongodb") status_set( "blocked", - "Ready for do-release-upgrade and reboot. " - "Set complete when finished.") + "Ready for do-release-upgrade and reboot. Set complete when finished.", + ) -@hooks.hook('post-series-upgrade') +@hooks.hook("post-series-upgrade") def post_series_upgrade(): juju_log("Running complete series upgrade hook", "INFO") update_status() @@ -1872,4 +1882,4 @@ if __name__ == "__main__": try: hooks.execute(sys.argv) except UnregisteredHookError as e: - juju_log('Unknown hook {} - skipping'.format(e)) + juju_log("Unknown hook {} - skipping".format(e)) diff --git a/tests/functional/tests/tests_mongodb.py b/tests/functional/tests/tests_mongodb.py index dc50d8d..bf80b65 100644 --- a/tests/functional/tests/tests_mongodb.py +++ b/tests/functional/tests/tests_mongodb.py @@ -115,7 +115,8 @@ class ReplicatedMongodbCharmTest(MongodbCharmTestBase): ] def extract(member_dict): - # Extract a subset of membership info as a frozenset. Name is ipaddr:port, health a float and stateStr a str + # Extract a subset of membership info as a frozenset. + # Name is ipaddr:port, health a float and stateStr a str return frozenset( v for k, v in member_dict.items() if k in ["name", "health", "stateStr"] ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 5dd2748..e515754 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,3 +1,4 @@ import sys -sys.path.append('hooks') -sys.path.append('unit_tests') + +sys.path.append("hooks") +sys.path.append("unit_tests") diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index e97ef91..b036f06 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -17,17 +17,16 @@ import hooks # hooks.some_func(). Invoking the the interface change relations will cause # the hooks context to be created outside of the normal mockery. TO_PATCH = [ - 'relation_id', - 'relation_get', - 'relation_set', - 'unit_get', - 'juju_log', - 'config', + "relation_id", + "relation_get", + "relation_set", + "unit_get", + "juju_log", + "config", ] class MongoHooksTest(CharmTestCase): - def setUp(self): super(MongoHooksTest, self).setUp(hooks, TO_PATCH) @@ -38,23 +37,27 @@ class MongoHooksTest(CharmTestCase): self.config.side_effect = self.test_config.get self.relation_get.side_effect = self.test_relation.get - @patch.object(hooks, 'is_relation_made') - @patch.object(hooks, 'get_replicaset_status') - @patch.object(hooks, 'restart_mongod') - @patch.object(hooks, 'enable_replset') + @patch.object(hooks, "is_relation_made") + @patch.object(hooks, "get_replicaset_status") + @patch.object(hooks, "restart_mongod") + @patch.object(hooks, "enable_replset") # Note: patching the os.environ dictionary in-line here so there's no # additional parameter sent into the function - @patch.dict('os.environ', JUJU_UNIT_NAME='fake-unit/0') - def test_replica_set_relation_joined(self, mock_enable_replset, - mock_restart, mock_get_replset_status, - mock_is_rel_made): - self.unit_get.return_value = 'private.address' - self.test_config.set('port', '1234') - self.test_config.set('replicaset', 'fake-replicaset') - self.relation_id.return_value = 'fake-relation-id' + @patch.dict("os.environ", JUJU_UNIT_NAME="fake-unit/0") + def test_replica_set_relation_joined( + self, + mock_enable_replset, + mock_restart, + mock_get_replset_status, + mock_is_rel_made, + ): + self.unit_get.return_value = "private.address" + self.test_config.set("port", "1234") + self.test_config.set("replicaset", "fake-replicaset") + self.relation_id.return_value = "fake-relation-id" mock_enable_replset.return_value = False - mock_get_replset_status.return_value = 'PRIMARY' + mock_get_replset_status.return_value = "PRIMARY" mock_is_rel_made.return_value = True hooks.replica_set_relation_joined() @@ -63,13 +66,15 @@ class MongoHooksTest(CharmTestCase): # was not enabled. self.assertFalse(mock_restart.called) - exp_rel_vals = {'hostname': 'private.address', - 'port': '1234', - 'replset': 'fake-replicaset', - 'install-order': '0', - 'type': 'replset'} + exp_rel_vals = { + "hostname": "private.address", + "port": "1234", + "replset": "fake-replicaset", + "install-order": "0", + "type": "replset", + } # Check that the relation data was set as we expect it to be set. - self.relation_set.assert_called_with('fake-relation-id', exp_rel_vals) + self.relation_set.assert_called_with("fake-relation-id", exp_rel_vals) mock_enable_replset.reset_mock() self.relation_set.reset_mock() @@ -78,24 +83,31 @@ class MongoHooksTest(CharmTestCase): hooks.replica_set_relation_joined() self.assertTrue(mock_restart.called) - self.relation_set.assert_called_with('fake-relation-id', exp_rel_vals) - - @patch.object(hooks, 'run_admin_command') - @patch.object(hooks, 'MongoClient') - @patch.object(hooks, 'config') - @patch.object(hooks, 'mongo_client') - @patch('time.sleep') - def test_init_repl_set(self, mock_sleep, mock_mongo_client_fn, - mock_config, mock_mongo_client, - mock_run_admin_command): + self.relation_set.assert_called_with("fake-relation-id", exp_rel_vals) + + @patch.object(hooks, "run_admin_command") + @patch.object(hooks, "MongoClient") + @patch.object(hooks, "config") + @patch.object(hooks, "mongo_client") + @patch("time.sleep") + def test_init_repl_set( + self, + mock_sleep, + mock_mongo_client_fn, + mock_config, + mock_mongo_client, + mock_run_admin_command, + ): mock_mongo_client_fn.return_value = False - mock_config.return_value = {'replicaset': 'foo', - 'private-address': 'mongo.local', - 'port': '12345'} + mock_config.return_value = { + "replicaset": "foo", + "private-address": "mongo.local", + "port": "12345", + } # Put the OK state (1) at the end and check the loop. - ret_values = [{'myState': x} for x in [0, 2, 5, 1]] + ret_values = [{"myState": x} for x in [0, 2, 5, 1]] mock_run_admin_command.side_effect = ret_values hooks.init_replset() @@ -104,8 +116,10 @@ class MongoHooksTest(CharmTestCase): self.assertEqual(len(ret_values) + 1, mock_sleep.call_count) mock_run_admin_command.reset_mock() - exc = [OperationFailure('Received replSetInitiate'), - OperationFailure('unhandled')] + exc = [ + OperationFailure("Received replSetInitiate"), + OperationFailure("unhandled"), + ] mock_run_admin_command.side_effect = exc try: @@ -116,78 +130,82 @@ class MongoHooksTest(CharmTestCase): self.assertEqual(2, mock_run_admin_command.call_count) - @patch.object(hooks, 'run') - @patch.object(hooks, 'juju_log') - @patch.object(hooks, 'is_bionic_or_greater') - @patch.object(hooks, 'mongo_client_smart') - def test_join_replset(self, mock_mongo_client, mock_is_bionic_or_greater, mock_juju_log, mock_run): + @patch.object(hooks, "run") + @patch.object(hooks, "juju_log") + @patch.object(hooks, "is_bionic_or_greater") + @patch.object(hooks, "mongo_client_smart") + def test_join_replset( + self, mock_mongo_client, mock_is_bionic_or_greater, mock_juju_log, mock_run + ): # Test with OS version not bionic or greater first. mock_is_bionic_or_greater.return_value = False hooks.join_replset() self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.join_replset(master_node='mongo.local') + hooks.join_replset(master_node="mongo.local") self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.join_replset(host='fake-host') + hooks.join_replset(host="fake-host") self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.join_replset(master_node='mongo.local', host='fake-host') - mock_mongo_client.assert_called_with('localhost', - 'rs.add("fake-host")') + hooks.join_replset(master_node="mongo.local", host="fake-host") + mock_mongo_client.assert_called_with("localhost", 'rs.add("fake-host")') # Also test with bionic or greater. old_mcr = hooks.MONGO_CLIENT_RETRIES hooks.MONGO_CLIENT_RETRIES = 0 mock_is_bionic_or_greater.return_value = True mock_mongo_client.reset_mock() - hooks.join_replset(master_node='mongo.local', host='fake-host') - expected_run = ['mongo', '--quiet', '--host', "localhost", '--eval', 'printjson(rs.add("fake-host"))'] + hooks.join_replset(master_node="mongo.local", host="fake-host") + expected_run = [ + "mongo", + "--quiet", + "--host", + "localhost", + "--eval", + 'printjson(rs.add("fake-host"))', + ] mock_run.assert_called_with(expected_run) # Restore mongo client retries for other tests. hooks.MONGO_CLIENT_RETRIES = old_mcr - @patch.object(hooks, 'mongo_client') + @patch.object(hooks, "mongo_client") def test_leave_replset(self, mock_mongo_client): hooks.leave_replset() self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.leave_replset(master_node='mongo.local') + hooks.leave_replset(master_node="mongo.local") self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.leave_replset(host='fake-host') + hooks.leave_replset(host="fake-host") self.assertFalse(mock_mongo_client.called) mock_mongo_client.reset_mock() - hooks.leave_replset('mongo.local', 'fake-host') - mock_mongo_client.assert_called_with('mongo.local', - 'rs.remove("fake-host")') - - @patch.object(hooks, 'apt_install') - @patch.object(hooks, 'apt_update') - @patch.object(hooks, 'add_source') - @patch.dict('os.environ', CHARM_DIR='/tmp/charm/dir') - def test_install_hook(self, mock_add_source, mock_apt_update, - mock_apt_install): - self.test_config.set('source', 'fake-source') - self.test_config.set('key', 'fake-key') + hooks.leave_replset("mongo.local", "fake-host") + mock_mongo_client.assert_called_with("mongo.local", 'rs.remove("fake-host")') + + @patch.object(hooks, "apt_install") + @patch.object(hooks, "apt_update") + @patch.object(hooks, "add_source") + @patch.dict("os.environ", CHARM_DIR="/tmp/charm/dir") + def test_install_hook(self, mock_add_source, mock_apt_update, mock_apt_install): + self.test_config.set("source", "fake-source") + self.test_config.set("key", "fake-key") hooks.install_hook() - mock_add_source.assert_called_with('fake-source', 'fake-key') + mock_add_source.assert_called_with("fake-source", "fake-key") mock_apt_update.assert_called_with(fatal=True) - mock_apt_install.assert_called_with(packages=hooks.INSTALL_PACKAGES, - fatal=True) - - @patch.object(hooks, 'run_admin_command') - @patch.object(hooks, 'MongoClient') - @patch('time.sleep') - def test_am_i_primary(self, mock_sleep, mock_mongo_client, - mock_run_admin_cmd): - mock_run_admin_cmd.side_effect = [{'myState': x} for x in range(5)] + mock_apt_install.assert_called_with(packages=hooks.INSTALL_PACKAGES, fatal=True) + + @patch.object(hooks, "run_admin_command") + @patch.object(hooks, "MongoClient") + @patch("time.sleep") + def test_am_i_primary(self, mock_sleep, mock_mongo_client, mock_run_admin_cmd): + mock_run_admin_cmd.side_effect = [{"myState": x} for x in range(5)] expected_results = [True if x == 1 else False for x in range(5)] # Check expected return values each time... @@ -195,54 +213,52 @@ class MongoHooksTest(CharmTestCase): rv = hooks.am_i_primary() self.assertEqual(exp, rv) - @patch.object(hooks, 'run_admin_command') - @patch.object(hooks, 'MongoClient') - @patch('time.sleep') - def test_am_i_primary_too_many_attempts(self, mock_sleep, - mock_mongo_client, - mock_run_admin_cmd): - msg = 'replSetInitiate - should come online shortly' - mock_run_admin_cmd.side_effect = [OperationFailure(msg) - for x in range(10)] + @patch.object(hooks, "run_admin_command") + @patch.object(hooks, "MongoClient") + @patch("time.sleep") + def test_am_i_primary_too_many_attempts( + self, mock_sleep, mock_mongo_client, mock_run_admin_cmd + ): + msg = "replSetInitiate - should come online shortly" + mock_run_admin_cmd.side_effect = [OperationFailure(msg) for x in range(10)] try: hooks.am_i_primary() - self.assertTrue(False, 'Expected failure.') + self.assertTrue(False, "Expected failure.") except hooks.TimeoutException: self.assertEqual(mock_run_admin_cmd.call_count, 10) pass - @patch.object(hooks, 'run_admin_command') - @patch.object(hooks, 'MongoClient') - @patch('time.sleep') - def test_am_i_primary_operation_failures(self, mock_sleep, - mock_mongo_client, - mock_run_admin_cmd): - msg = 'EMPTYCONFIG' + @patch.object(hooks, "run_admin_command") + @patch.object(hooks, "MongoClient") + @patch("time.sleep") + def test_am_i_primary_operation_failures( + self, mock_sleep, mock_mongo_client, mock_run_admin_cmd + ): + msg = "EMPTYCONFIG" mock_run_admin_cmd.side_effect = OperationFailure(msg) rv = hooks.am_i_primary() self.assertTrue(mock_run_admin_cmd.called) self.assertFalse(rv) mock_run_admin_cmd.reset_mock() - msg = 'not running with --replSet' + msg = "not running with --replSet" mock_run_admin_cmd.side_effect = OperationFailure(msg) rv = hooks.am_i_primary() self.assertTrue(mock_run_admin_cmd.called) self.assertFalse(rv) mock_run_admin_cmd.reset_mock() - mock_run_admin_cmd.side_effect = OperationFailure('unexpected failure') + mock_run_admin_cmd.side_effect = OperationFailure("unexpected failure") try: hooks.am_i_primary() self.assertFalse(True, "Expected OperationFailure to be raised") except OperationFailure: self.assertTrue(mock_run_admin_cmd.called) - @patch('time.sleep') - @patch('subprocess.check_output') - def test_mongo_client_smart_no_command(self, mock_check_output, - mock_sleep): + @patch("time.sleep") + @patch("subprocess.check_output") + def test_mongo_client_smart_no_command(self, mock_check_output, mock_sleep): rv = hooks.mongo_client_smart() self.assertFalse(rv) self.assertEqual(0, mock_check_output.call_count) @@ -250,74 +266,82 @@ class MongoHooksTest(CharmTestCase): mock_check_output.reset_mock() mock_check_output.return_value = b'{"ok": 1}' - rv = hooks.mongo_client_smart(command='fake-cmd') + rv = hooks.mongo_client_smart(command="fake-cmd") self.assertTrue(rv) - mock_check_output.assert_called_once_with(['mongo', '--quiet', - '--host', 'localhost', - '--eval', - 'printjson(fake-cmd)']) + mock_check_output.assert_called_once_with( + ["mongo", "--quiet", "--host", "localhost", "--eval", "printjson(fake-cmd)"] + ) - @patch('time.sleep') - @patch('subprocess.check_output') + @patch("time.sleep") + @patch("subprocess.check_output") def test_mongo_client_smart_error_cases(self, mock_ck_output, mock_sleep): - mock_ck_output.side_effect = [CalledProcessError(1, 'cmd', - output='fake-error') - for x in range(11)] - rv = hooks.mongo_client_smart(command='fake-cmd') + mock_ck_output.side_effect = [ + CalledProcessError(1, "cmd", output="fake-error") for x in range(11) + ] + rv = hooks.mongo_client_smart(command="fake-cmd") self.assertFalse(rv) - @patch('subprocess.call') + @patch("subprocess.call") def test_mongo_client(self, mock_subprocess): rv = hooks.mongo_client() self.assertFalse(rv) self.assertEqual(0, mock_subprocess.call_count) mock_subprocess.reset_mock() - rv = hooks.mongo_client(host='fake-host') + rv = hooks.mongo_client(host="fake-host") self.assertFalse(rv) self.assertEqual(0, mock_subprocess.call_count) mock_subprocess.reset_mock() - rv = hooks.mongo_client(command='fake-command') + rv = hooks.mongo_client(command="fake-command") self.assertFalse(rv) self.assertEqual(0, mock_subprocess.call_count) mock_subprocess.reset_mock() mock_subprocess.return_value = 0 - rv = hooks.mongo_client(host='fake-host', command='fake-command') - expected_cmd = ("mongo --host %s --eval 'printjson(%s)'" - % ('fake-host', 'fake-command')) + rv = hooks.mongo_client(host="fake-host", command="fake-command") + expected_cmd = "mongo --host %s --eval 'printjson(%s)'" % ( + "fake-host", + "fake-command", + ) mock_subprocess.assert_called_once_with(expected_cmd, shell=True) self.assertTrue(rv) mock_subprocess.reset_mock() mock_subprocess.return_value = 1 - rv = hooks.mongo_client(host='fake-host', command='fake-command') - expected_cmd = ("mongo --host %s --eval 'printjson(%s)'" - % ('fake-host', 'fake-command')) + rv = hooks.mongo_client(host="fake-host", command="fake-command") + expected_cmd = "mongo --host %s --eval 'printjson(%s)'" % ( + "fake-host", + "fake-command", + ) mock_subprocess.assert_called_once_with(expected_cmd, shell=True) self.assertFalse(rv) - @patch.object(hooks, 'is_relation_made') - @patch.object(hooks, 'run_admin_command') - @patch.object(hooks, 'is_leader') - @patch.object(hooks, 'get_replicaset_status') - @patch.object(hooks, 'am_i_primary') - @patch.object(hooks, 'init_replset') - @patch.object(hooks, 'relation_get') - @patch.object(hooks, 'peer_units') - @patch.object(hooks, 'join_replset') - @patch.object(hooks, 'unit_get') - def test_replica_set_relation_changed(self, mock_unit_get, - mock_join_replset, mock_peer_units, - mock_relation_get, mock_init_replset, - mock_is_primary, - mock_get_replset_status, - mock_is_leader, - mock_run_admin_cmd, - mock_is_rel_made): + @patch.object(hooks, "is_relation_made") + @patch.object(hooks, "run_admin_command") + @patch.object(hooks, "is_leader") + @patch.object(hooks, "get_replicaset_status") + @patch.object(hooks, "am_i_primary") + @patch.object(hooks, "init_replset") + @patch.object(hooks, "relation_get") + @patch.object(hooks, "peer_units") + @patch.object(hooks, "join_replset") + @patch.object(hooks, "unit_get") + def test_replica_set_relation_changed( + self, + mock_unit_get, + mock_join_replset, + mock_peer_units, + mock_relation_get, + mock_init_replset, + mock_is_primary, + mock_get_replset_status, + mock_is_leader, + mock_run_admin_cmd, + mock_is_rel_made, + ): # set the unit_get('private-address') - mock_unit_get.return_value = 'juju-local-unit-0.local' + mock_unit_get.return_value = "juju-local-unit-0.local" mock_relation_get.return_value = None # Test when remote hostname is None, should not join @@ -326,10 +350,10 @@ class MongoHooksTest(CharmTestCase): # Test remote hostname is valid, but master is somehow not defined mock_join_replset.reset_mock() - mock_relation_get.return_value = 'juju-local-unit-0' + mock_relation_get.return_value = "juju-local-unit-0" mock_is_leader.return_value = False - mock_run_admin_cmd.return_value = {'myState': hooks.MONGO_PRIMARY} - mock_get_replset_status.return_value = 'PRIMARY' + mock_run_admin_cmd.return_value = {"myState": hooks.MONGO_PRIMARY} + mock_get_replset_status.return_value = "PRIMARY" mock_is_rel_made.return_value = True hooks.replica_set_relation_changed() @@ -339,7 +363,7 @@ class MongoHooksTest(CharmTestCase): # Test when not oldest peer, don't init replica set mock_join_replset.reset_mock() mock_init_replset.reset_mock() - mock_peer_units.return_value = ['mongodb/1', 'mongodb/2'] + mock_peer_units.return_value = ["mongodb/1", "mongodb/2"] hooks.replica_set_relation_changed() @@ -347,21 +371,21 @@ class MongoHooksTest(CharmTestCase): # Test when its also the PRIMARY mock_relation_get.reset_mock() - mock_relation_get.side_effect = ['juju-remote-unit-0', '12345'] + mock_relation_get.side_effect = ["juju-remote-unit-0", "12345"] mock_is_primary.reset_mock() mock_is_primary.return_value = True mock_join_replset.reset_mock() hooks.replica_set_relation_changed() - call1 = call('juju-local-unit-0.local:27017', - 'juju-remote-unit-0:12345') + call1 = call("juju-local-unit-0.local:27017", "juju-remote-unit-0:12345") mock_join_replset.assert_has_calls([call1]) - @patch.object(hooks, 'unit_get') - @patch.object(hooks, 'leave_replset') - @patch.object(hooks, 'am_i_primary') - def test_replica_set_relation_departed(self, mock_am_i_primary, - mock_leave_replset, mock_unit_get): + @patch.object(hooks, "unit_get") + @patch.object(hooks, "leave_replset") + @patch.object(hooks, "am_i_primary") + def test_replica_set_relation_departed( + self, mock_am_i_primary, mock_leave_replset, mock_unit_get + ): mock_am_i_primary.return_value = False hooks.replica_set_relation_departed() @@ -369,25 +393,29 @@ class MongoHooksTest(CharmTestCase): mock_am_i_primary.reset_mock() mock_am_i_primary.return_value = True - mock_unit_get.return_value = 'juju-local' + mock_unit_get.return_value = "juju-local" - self.test_relation.set({'hostname': 'juju-remote', - 'port': '27017'}) + self.test_relation.set({"hostname": "juju-remote", "port": "27017"}) mock_leave_replset.reset_mock() hooks.replica_set_relation_departed() - call1 = call('juju-local:27017', 'juju-remote:27017') + call1 = call("juju-local:27017", "juju-remote:27017") mock_leave_replset.assert_has_calls([call1]) - @patch('time.sleep') - @patch.object(hooks, 'MongoClient') - @patch.object(hooks, 'unit_get') - @patch.object(hooks, 'leave_replset') - @patch.object(hooks, 'am_i_primary') - def test_replica_set_relation_broken(self, mock_am_i_primary, - mock_leave_replset, mock_unit_get, - mock_MongoClient, mock_sleep): + @patch("time.sleep") + @patch.object(hooks, "MongoClient") + @patch.object(hooks, "unit_get") + @patch.object(hooks, "leave_replset") + @patch.object(hooks, "am_i_primary") + def test_replica_set_relation_broken( + self, + mock_am_i_primary, + mock_leave_replset, + mock_unit_get, + mock_MongoClient, + mock_sleep, + ): mock_am_i_primary.return_value = False hooks.replica_set_relation_broken() @@ -405,12 +433,8 @@ class MongoHooksTest(CharmTestCase): three =four """ - expected = { - 'key': 'value', - 'one': 'two', - 'three': 'four' - } - with mock_open('/etc/mongodb.conf', test_config): + expected = {"key": "value", "one": "two", "three": "four"} + with mock_open("/etc/mongodb.conf", test_config): results = hooks.get_current_mongo_config() self.assertEqual(results, expected) @@ -438,14 +462,16 @@ class MongoHooksTest(CharmTestCase): os.unlink(mocked_upstart.name) @patch("subprocess.call") - @patch.object(hooks, 'is_relation_made') - @patch.object(hooks, 'is_bionic_or_greater') - def test_mongodb_conf(self, mock_is_bionic_or_greater, mock_is_relation_made, *args): + @patch.object(hooks, "is_relation_made") + @patch.object(hooks, "is_bionic_or_greater") + def test_mongodb_conf( + self, mock_is_bionic_or_greater, mock_is_relation_made, *args + ): mock_is_bionic_or_greater.return_value = False mock_is_relation_made.return_value = False tmpdir = tempfile.mkdtemp() - self.test_config.set('dbpath', os.path.join(tmpdir, 'db')) - self.test_config.set('logpath', os.path.join(tmpdir, 'log')) + self.test_config.set("dbpath", os.path.join(tmpdir, "db")) + self.test_config.set("logpath", os.path.join(tmpdir, "log")) try: mongodb_conf = hooks.mongodb_conf(self.test_config.get_all()) finally: @@ -470,18 +496,22 @@ diaglog = 0 rest = true master = true -""".format(tmpdir=tmpdir) +""".format( + tmpdir=tmpdir + ) self.assertEqual(mongodb_conf, expected) @patch("subprocess.call") - @patch.object(hooks, 'is_relation_made') - @patch.object(hooks, 'is_bionic_or_greater') - def test_mongodb_conf_bionic(self, mock_is_bionic_or_greater, mock_is_relation_made, *args): + @patch.object(hooks, "is_relation_made") + @patch.object(hooks, "is_bionic_or_greater") + def test_mongodb_conf_bionic( + self, mock_is_bionic_or_greater, mock_is_relation_made, *args + ): mock_is_bionic_or_greater.return_value = True mock_is_relation_made.return_value = False tmpdir = tempfile.mkdtemp() - self.test_config.set('dbpath', os.path.join(tmpdir, 'db')) - self.test_config.set('logpath', os.path.join(tmpdir, 'log')) + self.test_config.set("dbpath", os.path.join(tmpdir, "db")) + self.test_config.set("logpath", os.path.join(tmpdir, "log")) try: mongodb_conf = hooks.mongodb_conf(self.test_config.get_all()) finally: @@ -502,5 +532,7 @@ port = 27017 journal=true master = true -""".format(tmpdir=tmpdir) +""".format( + tmpdir=tmpdir + ) self.assertEqual(mongodb_conf, expected) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 583cce2..627f6bd 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10,48 +10,49 @@ from mock import patch @contextmanager def mock_open(filename, contents=None): - ''' Slightly simpler mock of open to return contents for filename ''' + """ Slightly simpler mock of open to return contents for filename """ + def mock_file(*args): if args[0] == filename: return io.StringIO(contents) else: return open(*args) - with patch('builtins.open', mock_file): + + with patch("builtins.open", mock_file): yield def load_config(): - ''' + """ Walk backwords from __file__ looking for config.yaml, load and return the 'options' section' - ''' + """ config = None f = __file__ while config is None: d = os.path.dirname(f) - if os.path.isfile(os.path.join(d, 'config.yaml')): - config = os.path.join(d, 'config.yaml') + if os.path.isfile(os.path.join(d, "config.yaml")): + config = os.path.join(d, "config.yaml") break f = d if not config: - logging.error('Could not find config.yaml in any parent directory ' - 'of %s. ' % f) + logging.error("Could not find config.yaml in any parent directory of %s." % f) raise Exception - return yaml.safe_load(open(config).read())['options'] + return yaml.safe_load(open(config).read())["options"] def get_default_config(): - ''' + """ Load default charm config from config.yaml return as a dict. If no default is set in config.yaml, its value is None. - ''' + """ default_config = {} config = load_config() for k, v in config.items(): - if 'default' in v: - default_config[k] = v['default'] + if "default" in v: + default_config[k] = v["default"] else: default_config[k] = None return default_config diff --git a/tests/unit/test_write_log_rotate_config.py b/tests/unit/test_write_log_rotate_config.py index ef1cd98..aa93772 100644 --- a/tests/unit/test_write_log_rotate_config.py +++ b/tests/unit/test_write_log_rotate_config.py @@ -6,28 +6,27 @@ import hooks class TestWriteLogrotateConfigFile(unittest.TestCase): - def test_success(self): - logpath = '/tmp/foo/foo.log' + logpath = "/tmp/foo/foo.log" config_data = { - 'logpath': logpath, - 'logrotate-frequency': 'daily', - 'logrotate-maxsize': '5G', - 'logrotate-rotate': 5, + "logpath": logpath, + "logrotate-frequency": "daily", + "logrotate-maxsize": "5G", + "logrotate-rotate": 5, } fd, temp_fn = tempfile.mkstemp() os.close(fd) - with mock.patch('hooks.juju_log') as mock_juju_log: - with mock.patch('hooks.open', create=True) as mock_open: + with mock.patch("hooks.juju_log") as mock_juju_log: + with mock.patch("hooks.open", create=True) as mock_open: mock_open.return_value = mock.MagicMock() hooks.write_logrotate_config(config_data, temp_fn) os.unlink(temp_fn) - mock_juju_log.assert_called_once_with('Writing {}.'.format(temp_fn)) - mock_open.assert_called_once_with(temp_fn, 'w') + mock_juju_log.assert_called_once_with("Writing {}.".format(temp_fn)) + mock_open.assert_called_once_with(temp_fn, "w") mock_file = mock_open().__enter__() call_args = mock_file.write.call_args[0][0] self.assertTrue(mock_file.write.called) self.assertIn(logpath, call_args) - self.assertIn('daily', call_args) - self.assertIn('maxsize 5G', call_args) - self.assertIn('rotate 5', call_args) + self.assertIn("daily", call_args) + self.assertIn("maxsize 5G", call_args) + self.assertIn("rotate 5", call_args) @@ -25,7 +25,7 @@ passenv = [testenv:lint] commands = flake8 -#TODO black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod)/" . + black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod)/" . deps = black flake8 @@ -35,7 +35,7 @@ deps = flake8-colors [flake8] -ignore = E402,E226,W504 +ignore = E402,E226,W503,W504 exclude = .git, __pycache__, @@ -44,8 +44,7 @@ exclude = mod, .build -#TODO max-line-length = 88 -max-line-length = 120 +max-line-length = 88 max-complexity = 10 [testenv:unit] |