Makefile.am | 2 +- src/lib389/lib389/perftools.py | 568 ------------------------------- src/lib389/lib389/perftools/__init__.py | 568 +++++++++++++++++++++++++++++++ src/lib389/lib389/topologies.py | 561 ------------------------------ src/lib389/lib389/topologies/__init__.py | 561 ++++++++++++++++++++++++++++++ src/lib389/setup.py | 11 +- 6 files changed, 1140 insertions(+), 1131 deletions(-) diff --git a/Makefile.am b/Makefile.am index b34947869..f3a7da85c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1167,7 +1167,7 @@ libslapd_la_SOURCES = ldap/servers/slapd/add.c \ $(libavl_a_SOURCES) libslapd_la_CPPFLAGS = $(AM_CPPFLAGS) $(DSPLUGIN_CPPFLAGS) $(SASL_CFLAGS) $(DB_INC) $(KERBEROS_CFLAGS) $(PCRE_CFLAGS) $(SVRCORE_INCLUDES) -libslapd_la_LIBADD = $(LDAPSDK_LINK) $(SASL_LINK) $(NSS_LINK) $(NSPR_LINK) $(KERBEROS_LIBS) $(PCRE_LIBS) $(THREADLIB) $(SYSTEMD_LIBS) libsvrcore.la $(RSLAPD_LIB) +libslapd_la_LIBADD = $(LDAPSDK_LINK) $(SASL_LINK) $(NSS_LINK) $(NSPR_LINK) $(KERBEROS_LIBS) $(PCRE_LIBS) $(THREADLIB) $(SYSTEMD_LIBS) libsvrcore.la $(RSLAPD_LIB) -lm # If asan is enabled, it creates special libcrypt interceptors. However, they are # detected by the first load of libasan at runtime, and what is in the linked lib # so we need libcrypt to be present as soon as libasan is loaded for the interceptors diff --git a/src/lib389/lib389/perftools.py b/src/lib389/lib389/perftools.py deleted file mode 100644 index 078acb015..000000000 --- a/src/lib389/lib389/perftools.py +++ /dev/null @@ -1,568 +0,0 @@ -#!/usr/bin/python3 - -# --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2021 Red Hat, Inc. -# All rights reserved. -# -# License: GPL (version 3 or any later version). -# See LICENSE for details. -# --- END COPYRIGHT BLOCK --- - -#import os -#import os.path -#import ldap -import sys -import re -import string -import logging -import subprocess -import multiprocessing -import time -import json -import statistics -from random import shuffle, seed, randint, choice -from lib389._constants import * -from lib389.properties import * -from lib389.idm.user import UserAccounts -from lib389.topologies import create_topology -from lib389 import DirSrv -from lib389.config import LMDB_LDBMConfig -from lib389.utils import get_default_db_lib -from pathlib import Path, PosixPath - -class IdGenerator: - # Generates up to nbids unique identifiers - - def __init__(self, nbids): - self._idx = 0 - self._nbids = nbids - - def __iter__(self): - self._idx = 0 - return self - - def _idFromIndex(self): - return self._formatId() - - def _formatId(self, id): - # Should be overwritten in subclass - # Should return an id - return str(id) - - def __next__(self): - if (self._idx >= self._nbids): - raise StopIteration - self._idx += 1 - return self._formatId(self._idx-1) - - def getIdx(self): - return self._idx - - def random(self): - self._idx = randint(0, self._nbids-1); - return self._formatId(self._idx) - -class IdGeneratorWithNames(IdGenerator): - # Generates up to nbids unique identifiers as names - # for perf reason a small (self._list_size) number of unique identifier is generated - # then ids from this list are concat level time (insuring the final id is unique) - - # Generates up to self._list_size power level unique identifiers - # for perf reason a small (self._list_size) number of unique identifier is generated - # then ids from this list are concat level time (insuring the final id is unique) - def __init__(self, nbids): - super().__init__(nbids) - self._voyelles = [ 'a', 'e', 'i', 'o', 'u', 'ai', 'an', 'au', 'en', 'ei', 'en', 'eu', 'in', 'on', 'ou' ] - self._consonnes = [ 'b', 'c', 'ch', 'cr', 'd', 'f', 'g', 'j', 'l', 'm', 'n', 'p', 'ph', 'qu', 'r', 's', 't', 'v' ] - self._syllabs = [c+v for c in self._consonnes for v in self._voyelles] - shuffle(self._syllabs) - self._level = 0 - self._syllabsLen = len(self._syllabs) - while (nbids > 0): - self._level = self._level+1 - nbids = int (nbids / self._syllabsLen) - - def _formatId(self, idx): - id = "" - for i in range(self._level): - id += self._syllabs[int(idx % self._syllabsLen)] - idx /= self._syllabsLen - return id.capitalize() - -class IdGeneratorWithNumbers(IdGenerator): - # Generates up to nbids unique identifiers as numbers - - @staticmethod - def formatId(idx): - return f'{idx:0>10d}' - - def _formatId(self, idx): - return IdGeneratorWithNumbers.formatId(idx) - - -class CsvFile: - # Helper to write simple csv files - - def __init__(self, fname, width): - self.fname = fname - self.f = None - self.width = width - self.pos = 1 - self.line = "" - self.sep = ";" - self.lineid = 1 - - def __enter__(self): - self.f = open(self.fname, "w") if self.fname else sys.stdout - return self - - def __exit__(self, type, value, tb): - if (self.f != sys.stdout): - self.f.close() - self.f = None - - def nf(self, str): - if not str: - str="" - self.line += f"{str}{self.sep}" - self.pos += 1 - - def nl(self): - while self.pos < self.width: - self.nf(None) - self.line += "\n" - self.f.write(self.line) - self.line = "" - self.pos = 1 - self.lineid += 1 - - def n(self, v): - # Get name of spreadsheet column - if (v == 0): - return "" - return chr(0x40+v) - - def ref(self, dpl): - colid = self.pos + dpl - 1 - return f"{self.n(int(colid/26))}{self.n(colid%26+1)}{self.lineid}" - -class PerformanceTools: - - def __init__(self, options = {}): - prefix = os.path.join(os.environ.get('PREFIX', "")) - perfdir= f"{prefix}/var/log/dirsrv/perfdir" - print(f"Results and logs are stored in {perfdir} directory.") - self._options = { - 'nbUsers' : 10000, - 'seed' : 'lib389PerfTools', - 'resultDir' : perfdir, - 'suffix' : DEFAULT_SUFFIX, - **options - } - seed(self._options['seed']) - self._instance = None - os.makedirs(perfdir, mode=0o755, exist_ok = True) - self._ldclt_template = self.getFilePath("template.ldclt"); - # Generate a dummy template anyway we do not plan to create entries - with open(self._ldclt_template, "w") as f: - f.write("objectclass: inetOrgPerson\n"); - self._users_parents_dn = f"ou=People,{self._options['suffix']}" - - @staticmethod - def log2Csv(fname, fout): - # Convert (verbose) log file into csv file (easier for comparing the results) - map={} # ( nb_users, name, nbthreads, db_lib) Tuple to Result map - names={} # { Name : None } Map - has_threads={} # { Name : { threads : { users : users } } } Map - # Read log file - maxmes=0 - with open(fname) as f: - for line in f: - if (line[0] != '{'): - continue - if (line[-1] == '\n'): - line = line[:-1] - res = eval(line.replace('\n','\\n')) - nb_users = res['nbUsers'] - db_lib = res['db_lib'] - name = res['measure_name'] - names[name] = None - try: - nbthreads = res['nb_threads'] - except KeyError: - nbthreads = "" - if not name in has_threads: - has_threads[name] = {} - if not nbthreads in has_threads[name]: - has_threads[name][nbthreads] = {} - has_threads[name][nbthreads][nb_users] = nb_users - key = ( nb_users, name, nbthreads, db_lib) - if not key in map: - map[key] = [] - m = map[key] - m.append(res) - if maxmes < len(m): - maxmes = len(m) - # Displays the result: by test name then by thread number then by users number - # Generates all combinations - keys=[] - for name in sorted(names.keys()): - for t in sorted(has_threads[name].keys()): - for user in sorted(has_threads[name][t].keys()): - keys.append((user, name, t)) - #Generates the csv file - sep=";" - with CsvFile(fout, 2*maxmes + 2) as csv: - csv.nf("test name"); - csv.nf("threads"); - csv.nf("users"); - for idx in range(maxmes): - csv.nf("bdb"); - csv.nf("mdb"); - csv.nf("%"); - csv.nl(); - for k in keys: - csv.nf(f"{k[1]}") - csv.nf(f"{k[2]}") - csv.nf(f"{k[0]}") - k0 = ( k[0], k[1], k[2], "bdb" ) - k1 = ( k[0], k[1], k[2], "mdb" ) - for idx in range(maxmes): - if k0 in map and idx < len(map[k0]): - res = map[k0][idx] - csv.nf(res['safemean']) - else: - csv.nf(None) - if k1 in map and idx < len(map[k1]): - res = map[k1][idx] - csv.nf(res['safemean']) - else: - csv.nf(None) - # Add % formula - csv.nf(f"=({csv.ref(-1)}-{csv.ref(-2)})/{csv.ref(-2)}") - csv.nl(); - - def getFilePath(self, filename): - return os.path.join(self._options['resultDir'], filename) - - def log(self, filename, msg): - with open(self.getFilePath(filename), "at") as f: - f.write(str(msg)) - f.write("\n") - - def initInstance(self): - if (self._instance): - return self._instance; - uidpath = self.getFilePath("uids") - nb_uids = 0 - try: - with open(uidpath, 'r') as f: - while f.readline(): - nb_uids += 1 - except FileNotFoundError: - pass - nb_users = self._options['nbUsers'] - need_rebuild = True - if (nb_uids == nb_users): - # Lets try to reuse existing instance - try : - self._instance = DirSrv(verbose=True) - self._instance.local_simple_allocate(serverid="standalone1", password=PW_DM) - self._instance.open() - if (self._instance.exists()): - if (self._instance.get_db_lib() == get_default_db_lib()): - need_rebuild = False - else: - print (f"db is {self._instance.get_db_lib()} instead of {get_default_db_lib()} ==> instance must be rebuild") - else: - print (f"missing instance ==> instance must be rebuild") - except Exception: - pass - else: - print (f"Instance has {nb_uids} users instead of {nb_users} ==> instance must be rebuild") - if (need_rebuild): - print ("Rebuilding standalone1 instance") - # Should rebuild the instance from scratch - topology = create_topology({ReplicaRole.STANDALONE: 1}) - self._instance = topology.standalone - # Adjust db size if needed (i.e about 670 K users) - defaultDBsize = 1073741824 - entrySize = 2000 # Real size is around 1525 but got error with 1800 (likely due to some recent changes in entries) - if (self._instance.get_db_lib() == "mdb" and - nb_users * entrySize > defaultDBsize): - mdb_config = LMDB_LDBMConfig(self._instance) - mdb_config.replace("nsslapd-mdb-max-size", str(nb_users * entrySize)) - self._instance.restart() - # Then populate the users - useraccounts = UserAccounts(self._instance, self._options['suffix']) - with open(uidpath, 'w') as f: - uidgen = IdGeneratorWithNumbers(nb_users) - cnGen = IdGeneratorWithNames(100) - snGen = IdGeneratorWithNames(100) - - for uid in uidgen: - cn = cnGen.random() - sn = snGen.random() - rdn = f"uid={uid}" - osuid = uidgen.getIdx() + 1000 - osgid = int (osuid % 100) + 1000 - properties = { - 'uid': uid, - 'cn': cn, - 'sn': sn, - 'uidNumber': str(osuid), - 'gidNumber': str(osgid), - 'homeDirectory': f'/home/{uid}' - } - super(UserAccounts, useraccounts).create(rdn, properties) - f.write(f'{uid}\n') - return self._instance; - - @staticmethod - def filterMeasures(values, m, ecart): - # keep values around m - r = [] - for val in values: - if (val > (1 - ecart) * m and val < (1 + ecart) * m): - r.append(val) - return r - - def safeMeasures(self, values, ecart=0.2): - v = values - try: - r = PerformanceTools.filterMeasures(values, statistics.mean(v) , ecart) - while ( r != v ): - v = r - r = PerformanceTools.filterMeasures(values, statistics.mean(v) , ecart) - if (len(r) == 0): - return values - return r - except statistics.StatisticsError as e: - self.log("log", str(e)) - print(e) - return values - - # Return a dict about the evironment data - def getEnvInfo(self): - mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024.**3) - with open ('/etc/redhat-release') as f: - release = f.read() - return { - "db_lib" : self._instance.get_db_lib(), - "nb_cpus" : multiprocessing.cpu_count(), - "total mem" : mem, - "release" : str(release), - **self._options - } - - def finalizeResult(self, res): - try: - rawres = res["rawresults"] - res["rawmean"] = statistics.mean(rawres) - res["saferesults"] = self.safeMeasures(rawres) # discard first measure result - res["safemean"] = statistics.mean(res["saferesults"]) - pretty_res_keys = [ 'start_time', 'stop_time', 'measure_name', 'safemean', 'db_lib', 'nbUsers', 'nb_threads' ] - pretty_res = dict(filter(lambda elem: elem[0] in pretty_res_keys, res.items())) - except statistics.StatisticsError as e: - print(e) - res["exception"] = e - pretty_res = "#ERROR" - res["pretty"] = pretty_res - self.log("out", res["pretty"]) - self.log("log", res) - return res - - def ldclt(self, measure_name, args, nbThreads=10, nbMes=10): - # First ldclt measure is always bad so do 1 measure more - # and discard it from final result - nbMes += 1 - - prog = os.path.join(self._instance.ds_paths.bin_dir, 'ldclt') - cmd = [ prog, - '-h', - f'{self._instance.host}', - '-p', - f'{self._instance.port}', - '-D', - f'{self._instance.binddn}', - '-w', - f'{self._instance.bindpw}', - '-N', str(nbMes), - '-n', str(nbThreads) ] - for key in args.keys(): - cmd.append(str(key)) - val = args[key] - if (val): - cmd.append(str(val)) - start_time = time.time() - tmout = 30+10*nbMes - print (f"Running ldclt with a timeout of {tmout} seconds ...\r") - try: - result = subprocess.run(args=cmd, capture_output=True, timeout=tmout) - except subprocess.CalledProcessError as e: - self.log("log", f'{e.cmd} failed. measure: {measure_name}\n' + - f'instance: {self._instance.serverid}\n' + - f'return code is {e.returncode}.\n' + - f'stdout: {e.stdout}\n' + - f'stderr: {e.stderr}\n' ) - raise e - print (" Done.") - stop_time = time.time() - # Lets parse the result - res = { "measure_name" : measure_name, - "cmd" : cmd, - "stdout" : result.stdout, - "stderr" : result.stderr, - "returncode" : result.returncode, - "start_time" : start_time, - "stop_time" : stop_time, - "stop_time" : stop_time, - "nb_threads" : nbThreads, - **self.getEnvInfo() } - rawres = re.findall(r'Average rate: [^ ]*\s*.([^/]*)', str(result.stdout)) - rawres = [float(i) for i in rawres] - res["measure0"] = rawres[0] - res["rawresults"] = rawres[1:] # Discard first measure - return self.finalizeResult(res) - - def measure_search_by_uid(self, name, nb_threads = 1): - nb_users = self._options['nbUsers'] - args = { "-b" : self._users_parents_dn, - "-f" : "uid=XXXXXXXXXX", - "-e" : "esearch,random", - "-r0" : None, - f"-R{nb_users-1}" : None } - return self.ldclt(name, args, nbThreads=nb_threads) - - # I wish I could make the base dn vary rather than use the dn in filter - # but I did not find how to do that (the RDN trick as in modify - # generates the same search than measure_search_by_uid test) - def measure_search_by_filtering_the_dn(self, name, nb_threads = 1): - nb_users = self._options['nbUsers'] - args = { "-b" : self._users_parents_dn, - "-f" : "uid:dn:=XXXXXXXXXX", - "-e" : "esearch,random", - "-r0" : None, - f"-R{nb_users-1}" : None } - return self.ldclt(name, args, nbThreads=nb_threads) - - def measure_modify(self, name, nb_threads = 1): - nb_users = self._options['nbUsers'] - args = { "-b" : self._users_parents_dn, - "-e" : f"rdn=uid:[RNDN(0;{nb_users-1};10)],object={self._ldclt_template},attreplace=sn: random modify XXXXX" } - return self.ldclt(name, args, nbThreads=nb_threads) - - def offline_export(self): - start_time = time.time() - assert (self._instance.db2ldif(DEFAULT_BENAME, (self._options['suffix'],), None, None, None, self._ldif)) - stop_time = time.time() - # Count entries in ldif file (if not already done) - if not self._nbEntries: - self._nbEntries = 0 - with open(self._ldif) as f: - for line in f: - if (line.startswith("dn:")): - self._nbEntries += 1 - return self._nbEntries / (stop_time - start_time) - - def offline_import(self): - start_time = time.time() - assert (self._instance.ldif2db(DEFAULT_BENAME, None, None, None, self._ldif)) - stop_time = time.time() - return self._nbEntries / (stop_time - start_time) - - def _do_measure(self, measure_name, measure_cb, nbMes): - # Perform non ldcltl measure - # - first_time = time.time() - rawres = [] - for m in range(nbMes): - try: - rawres.append( measure_cb() ) - stop_time = time.time() - except AssertionError: - continue - last_time = time.time() - # Lets parse the result - res = { "measure_name" : measure_name, - "start_time" : first_time, - "stop_time" : last_time, - "nb_measures" : nbMes, - "rawresults" : rawres, - **self.getEnvInfo() } - return self.finalizeResult(res) - - def mesure_export_import(self, nbMes=10): - self._instance.stop() - self._ldif = self.getFilePath("db.ldif"); - self._nbEntries = None - res = [ self._do_measure("export", self.offline_export, nbMes), self._do_measure("import", self.offline_import, nbMes) ] - self._instance.start() - return res; - - class Tester: - # Basic tester (used to define ldclt tests) - def __init__(self, name, description, method_name): - self._base_name = name - self._base_description = description - self._method_name = method_name - - def name(self): - return self._base_name - - def argsused(self): - return [ "nb_threads", "name" ] - - def description(self): - return self._base_description - - def run(self, perftools, args): - args['name'] = self._base_name - res = getattr(perftools, self._method_name)(self._base_name, nb_threads=args['nb_threads']); - print (res['pretty']) - - @staticmethod - def initTester(args): - os.environ["NSSLAPD_DB_LIB"] = args['db_lib'] - perftools = PerformanceTools( args ) - perftools.initInstance() - return perftools; - - class TesterImportExport(Tester): - # A special tester for export/import - def __init__(self): - super().__init__("export/import", - "Measure export rate in entries per seconds then measure import rate.", - None) - - def argsused(self): - return [] - - def run(self, perftools, args=None): - res = perftools.mesure_export_import() - for r in res: - print (r['pretty']) - - @staticmethod - def listTests(): - # List of test for which args.nb_threads is useful - return { t.name() : t for t in [ - PerformanceTools.Tester("search_uid", "Measure number of searches per seconds using filter with random existing uid.", "measure_search_by_uid"), - PerformanceTools.Tester("search_uid_in_dn", "Measure number of searches per seconds using filter with random existing uid in dn (i.e: (uid:dn:uid_value)).", "measure_search_by_filtering_the_dn"), - PerformanceTools.Tester("modify_sn", "Measure number of modify per seconds replacing sn by random value on random entries.", "measure_modify"), - PerformanceTools.TesterImportExport(), - ] } - - @staticmethod - def runAllTests(options): - for users in ( 100, 1000, 10000, 100000, 1000000 ): - for db in ( 'bdb', 'mdb' ): - perftools = PerformanceTools.Tester.initTester({**options, 'nbUsers': users, 'db_lib': db}) - for t in PerformanceTools.listTests().values(): - if 'nb_threads' in t.argsused(): - for nbthreads in ( 1, 4, 8 ): - t.run(perftools, { "nb_threads" : nbthreads }) - else: - t.run(perftools) - - diff --git a/src/lib389/lib389/perftools/__init__.py b/src/lib389/lib389/perftools/__init__.py new file mode 100644 index 000000000..078acb015 --- /dev/null +++ b/src/lib389/lib389/perftools/__init__.py @@ -0,0 +1,568 @@ +#!/usr/bin/python3 + +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2021 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +#import os +#import os.path +#import ldap +import sys +import re +import string +import logging +import subprocess +import multiprocessing +import time +import json +import statistics +from random import shuffle, seed, randint, choice +from lib389._constants import * +from lib389.properties import * +from lib389.idm.user import UserAccounts +from lib389.topologies import create_topology +from lib389 import DirSrv +from lib389.config import LMDB_LDBMConfig +from lib389.utils import get_default_db_lib +from pathlib import Path, PosixPath + +class IdGenerator: + # Generates up to nbids unique identifiers + + def __init__(self, nbids): + self._idx = 0 + self._nbids = nbids + + def __iter__(self): + self._idx = 0 + return self + + def _idFromIndex(self): + return self._formatId() + + def _formatId(self, id): + # Should be overwritten in subclass + # Should return an id + return str(id) + + def __next__(self): + if (self._idx >= self._nbids): + raise StopIteration + self._idx += 1 + return self._formatId(self._idx-1) + + def getIdx(self): + return self._idx + + def random(self): + self._idx = randint(0, self._nbids-1); + return self._formatId(self._idx) + +class IdGeneratorWithNames(IdGenerator): + # Generates up to nbids unique identifiers as names + # for perf reason a small (self._list_size) number of unique identifier is generated + # then ids from this list are concat level time (insuring the final id is unique) + + # Generates up to self._list_size power level unique identifiers + # for perf reason a small (self._list_size) number of unique identifier is generated + # then ids from this list are concat level time (insuring the final id is unique) + def __init__(self, nbids): + super().__init__(nbids) + self._voyelles = [ 'a', 'e', 'i', 'o', 'u', 'ai', 'an', 'au', 'en', 'ei', 'en', 'eu', 'in', 'on', 'ou' ] + self._consonnes = [ 'b', 'c', 'ch', 'cr', 'd', 'f', 'g', 'j', 'l', 'm', 'n', 'p', 'ph', 'qu', 'r', 's', 't', 'v' ] + self._syllabs = [c+v for c in self._consonnes for v in self._voyelles] + shuffle(self._syllabs) + self._level = 0 + self._syllabsLen = len(self._syllabs) + while (nbids > 0): + self._level = self._level+1 + nbids = int (nbids / self._syllabsLen) + + def _formatId(self, idx): + id = "" + for i in range(self._level): + id += self._syllabs[int(idx % self._syllabsLen)] + idx /= self._syllabsLen + return id.capitalize() + +class IdGeneratorWithNumbers(IdGenerator): + # Generates up to nbids unique identifiers as numbers + + @staticmethod + def formatId(idx): + return f'{idx:0>10d}' + + def _formatId(self, idx): + return IdGeneratorWithNumbers.formatId(idx) + + +class CsvFile: + # Helper to write simple csv files + + def __init__(self, fname, width): + self.fname = fname + self.f = None + self.width = width + self.pos = 1 + self.line = "" + self.sep = ";" + self.lineid = 1 + + def __enter__(self): + self.f = open(self.fname, "w") if self.fname else sys.stdout + return self + + def __exit__(self, type, value, tb): + if (self.f != sys.stdout): + self.f.close() + self.f = None + + def nf(self, str): + if not str: + str="" + self.line += f"{str}{self.sep}" + self.pos += 1 + + def nl(self): + while self.pos < self.width: + self.nf(None) + self.line += "\n" + self.f.write(self.line) + self.line = "" + self.pos = 1 + self.lineid += 1 + + def n(self, v): + # Get name of spreadsheet column + if (v == 0): + return "" + return chr(0x40+v) + + def ref(self, dpl): + colid = self.pos + dpl - 1 + return f"{self.n(int(colid/26))}{self.n(colid%26+1)}{self.lineid}" + +class PerformanceTools: + + def __init__(self, options = {}): + prefix = os.path.join(os.environ.get('PREFIX', "")) + perfdir= f"{prefix}/var/log/dirsrv/perfdir" + print(f"Results and logs are stored in {perfdir} directory.") + self._options = { + 'nbUsers' : 10000, + 'seed' : 'lib389PerfTools', + 'resultDir' : perfdir, + 'suffix' : DEFAULT_SUFFIX, + **options + } + seed(self._options['seed']) + self._instance = None + os.makedirs(perfdir, mode=0o755, exist_ok = True) + self._ldclt_template = self.getFilePath("template.ldclt"); + # Generate a dummy template anyway we do not plan to create entries + with open(self._ldclt_template, "w") as f: + f.write("objectclass: inetOrgPerson\n"); + self._users_parents_dn = f"ou=People,{self._options['suffix']}" + + @staticmethod + def log2Csv(fname, fout): + # Convert (verbose) log file into csv file (easier for comparing the results) + map={} # ( nb_users, name, nbthreads, db_lib) Tuple to Result map + names={} # { Name : None } Map + has_threads={} # { Name : { threads : { users : users } } } Map + # Read log file + maxmes=0 + with open(fname) as f: + for line in f: + if (line[0] != '{'): + continue + if (line[-1] == '\n'): + line = line[:-1] + res = eval(line.replace('\n','\\n')) + nb_users = res['nbUsers'] + db_lib = res['db_lib'] + name = res['measure_name'] + names[name] = None + try: + nbthreads = res['nb_threads'] + except KeyError: + nbthreads = "" + if not name in has_threads: + has_threads[name] = {} + if not nbthreads in has_threads[name]: + has_threads[name][nbthreads] = {} + has_threads[name][nbthreads][nb_users] = nb_users + key = ( nb_users, name, nbthreads, db_lib) + if not key in map: + map[key] = [] + m = map[key] + m.append(res) + if maxmes < len(m): + maxmes = len(m) + # Displays the result: by test name then by thread number then by users number + # Generates all combinations + keys=[] + for name in sorted(names.keys()): + for t in sorted(has_threads[name].keys()): + for user in sorted(has_threads[name][t].keys()): + keys.append((user, name, t)) + #Generates the csv file + sep=";" + with CsvFile(fout, 2*maxmes + 2) as csv: + csv.nf("test name"); + csv.nf("threads"); + csv.nf("users"); + for idx in range(maxmes): + csv.nf("bdb"); + csv.nf("mdb"); + csv.nf("%"); + csv.nl(); + for k in keys: + csv.nf(f"{k[1]}") + csv.nf(f"{k[2]}") + csv.nf(f"{k[0]}") + k0 = ( k[0], k[1], k[2], "bdb" ) + k1 = ( k[0], k[1], k[2], "mdb" ) + for idx in range(maxmes): + if k0 in map and idx < len(map[k0]): + res = map[k0][idx] + csv.nf(res['safemean']) + else: + csv.nf(None) + if k1 in map and idx < len(map[k1]): + res = map[k1][idx] + csv.nf(res['safemean']) + else: + csv.nf(None) + # Add % formula + csv.nf(f"=({csv.ref(-1)}-{csv.ref(-2)})/{csv.ref(-2)}") + csv.nl(); + + def getFilePath(self, filename): + return os.path.join(self._options['resultDir'], filename) + + def log(self, filename, msg): + with open(self.getFilePath(filename), "at") as f: + f.write(str(msg)) + f.write("\n") + + def initInstance(self): + if (self._instance): + return self._instance; + uidpath = self.getFilePath("uids") + nb_uids = 0 + try: + with open(uidpath, 'r') as f: + while f.readline(): + nb_uids += 1 + except FileNotFoundError: + pass + nb_users = self._options['nbUsers'] + need_rebuild = True + if (nb_uids == nb_users): + # Lets try to reuse existing instance + try : + self._instance = DirSrv(verbose=True) + self._instance.local_simple_allocate(serverid="standalone1", password=PW_DM) + self._instance.open() + if (self._instance.exists()): + if (self._instance.get_db_lib() == get_default_db_lib()): + need_rebuild = False + else: + print (f"db is {self._instance.get_db_lib()} instead of {get_default_db_lib()} ==> instance must be rebuild") + else: + print (f"missing instance ==> instance must be rebuild") + except Exception: + pass + else: + print (f"Instance has {nb_uids} users instead of {nb_users} ==> instance must be rebuild") + if (need_rebuild): + print ("Rebuilding standalone1 instance") + # Should rebuild the instance from scratch + topology = create_topology({ReplicaRole.STANDALONE: 1}) + self._instance = topology.standalone + # Adjust db size if needed (i.e about 670 K users) + defaultDBsize = 1073741824 + entrySize = 2000 # Real size is around 1525 but got error with 1800 (likely due to some recent changes in entries) + if (self._instance.get_db_lib() == "mdb" and + nb_users * entrySize > defaultDBsize): + mdb_config = LMDB_LDBMConfig(self._instance) + mdb_config.replace("nsslapd-mdb-max-size", str(nb_users * entrySize)) + self._instance.restart() + # Then populate the users + useraccounts = UserAccounts(self._instance, self._options['suffix']) + with open(uidpath, 'w') as f: + uidgen = IdGeneratorWithNumbers(nb_users) + cnGen = IdGeneratorWithNames(100) + snGen = IdGeneratorWithNames(100) + + for uid in uidgen: + cn = cnGen.random() + sn = snGen.random() + rdn = f"uid={uid}" + osuid = uidgen.getIdx() + 1000 + osgid = int (osuid % 100) + 1000 + properties = { + 'uid': uid, + 'cn': cn, + 'sn': sn, + 'uidNumber': str(osuid), + 'gidNumber': str(osgid), + 'homeDirectory': f'/home/{uid}' + } + super(UserAccounts, useraccounts).create(rdn, properties) + f.write(f'{uid}\n') + return self._instance; + + @staticmethod + def filterMeasures(values, m, ecart): + # keep values around m + r = [] + for val in values: + if (val > (1 - ecart) * m and val < (1 + ecart) * m): + r.append(val) + return r + + def safeMeasures(self, values, ecart=0.2): + v = values + try: + r = PerformanceTools.filterMeasures(values, statistics.mean(v) , ecart) + while ( r != v ): + v = r + r = PerformanceTools.filterMeasures(values, statistics.mean(v) , ecart) + if (len(r) == 0): + return values + return r + except statistics.StatisticsError as e: + self.log("log", str(e)) + print(e) + return values + + # Return a dict about the evironment data + def getEnvInfo(self): + mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024.**3) + with open ('/etc/redhat-release') as f: + release = f.read() + return { + "db_lib" : self._instance.get_db_lib(), + "nb_cpus" : multiprocessing.cpu_count(), + "total mem" : mem, + "release" : str(release), + **self._options + } + + def finalizeResult(self, res): + try: + rawres = res["rawresults"] + res["rawmean"] = statistics.mean(rawres) + res["saferesults"] = self.safeMeasures(rawres) # discard first measure result + res["safemean"] = statistics.mean(res["saferesults"]) + pretty_res_keys = [ 'start_time', 'stop_time', 'measure_name', 'safemean', 'db_lib', 'nbUsers', 'nb_threads' ] + pretty_res = dict(filter(lambda elem: elem[0] in pretty_res_keys, res.items())) + except statistics.StatisticsError as e: + print(e) + res["exception"] = e + pretty_res = "#ERROR" + res["pretty"] = pretty_res + self.log("out", res["pretty"]) + self.log("log", res) + return res + + def ldclt(self, measure_name, args, nbThreads=10, nbMes=10): + # First ldclt measure is always bad so do 1 measure more + # and discard it from final result + nbMes += 1 + + prog = os.path.join(self._instance.ds_paths.bin_dir, 'ldclt') + cmd = [ prog, + '-h', + f'{self._instance.host}', + '-p', + f'{self._instance.port}', + '-D', + f'{self._instance.binddn}', + '-w', + f'{self._instance.bindpw}', + '-N', str(nbMes), + '-n', str(nbThreads) ] + for key in args.keys(): + cmd.append(str(key)) + val = args[key] + if (val): + cmd.append(str(val)) + start_time = time.time() + tmout = 30+10*nbMes + print (f"Running ldclt with a timeout of {tmout} seconds ...\r") + try: + result = subprocess.run(args=cmd, capture_output=True, timeout=tmout) + except subprocess.CalledProcessError as e: + self.log("log", f'{e.cmd} failed. measure: {measure_name}\n' + + f'instance: {self._instance.serverid}\n' + + f'return code is {e.returncode}.\n' + + f'stdout: {e.stdout}\n' + + f'stderr: {e.stderr}\n' ) + raise e + print (" Done.") + stop_time = time.time() + # Lets parse the result + res = { "measure_name" : measure_name, + "cmd" : cmd, + "stdout" : result.stdout, + "stderr" : result.stderr, + "returncode" : result.returncode, + "start_time" : start_time, + "stop_time" : stop_time, + "stop_time" : stop_time, + "nb_threads" : nbThreads, + **self.getEnvInfo() } + rawres = re.findall(r'Average rate: [^ ]*\s*.([^/]*)', str(result.stdout)) + rawres = [float(i) for i in rawres] + res["measure0"] = rawres[0] + res["rawresults"] = rawres[1:] # Discard first measure + return self.finalizeResult(res) + + def measure_search_by_uid(self, name, nb_threads = 1): + nb_users = self._options['nbUsers'] + args = { "-b" : self._users_parents_dn, + "-f" : "uid=XXXXXXXXXX", + "-e" : "esearch,random", + "-r0" : None, + f"-R{nb_users-1}" : None } + return self.ldclt(name, args, nbThreads=nb_threads) + + # I wish I could make the base dn vary rather than use the dn in filter + # but I did not find how to do that (the RDN trick as in modify + # generates the same search than measure_search_by_uid test) + def measure_search_by_filtering_the_dn(self, name, nb_threads = 1): + nb_users = self._options['nbUsers'] + args = { "-b" : self._users_parents_dn, + "-f" : "uid:dn:=XXXXXXXXXX", + "-e" : "esearch,random", + "-r0" : None, + f"-R{nb_users-1}" : None } + return self.ldclt(name, args, nbThreads=nb_threads) + + def measure_modify(self, name, nb_threads = 1): + nb_users = self._options['nbUsers'] + args = { "-b" : self._users_parents_dn, + "-e" : f"rdn=uid:[RNDN(0;{nb_users-1};10)],object={self._ldclt_template},attreplace=sn: random modify XXXXX" } + return self.ldclt(name, args, nbThreads=nb_threads) + + def offline_export(self): + start_time = time.time() + assert (self._instance.db2ldif(DEFAULT_BENAME, (self._options['suffix'],), None, None, None, self._ldif)) + stop_time = time.time() + # Count entries in ldif file (if not already done) + if not self._nbEntries: + self._nbEntries = 0 + with open(self._ldif) as f: + for line in f: + if (line.startswith("dn:")): + self._nbEntries += 1 + return self._nbEntries / (stop_time - start_time) + + def offline_import(self): + start_time = time.time() + assert (self._instance.ldif2db(DEFAULT_BENAME, None, None, None, self._ldif)) + stop_time = time.time() + return self._nbEntries / (stop_time - start_time) + + def _do_measure(self, measure_name, measure_cb, nbMes): + # Perform non ldcltl measure + # + first_time = time.time() + rawres = [] + for m in range(nbMes): + try: + rawres.append( measure_cb() ) + stop_time = time.time() + except AssertionError: + continue + last_time = time.time() + # Lets parse the result + res = { "measure_name" : measure_name, + "start_time" : first_time, + "stop_time" : last_time, + "nb_measures" : nbMes, + "rawresults" : rawres, + **self.getEnvInfo() } + return self.finalizeResult(res) + + def mesure_export_import(self, nbMes=10): + self._instance.stop() + self._ldif = self.getFilePath("db.ldif"); + self._nbEntries = None + res = [ self._do_measure("export", self.offline_export, nbMes), self._do_measure("import", self.offline_import, nbMes) ] + self._instance.start() + return res; + + class Tester: + # Basic tester (used to define ldclt tests) + def __init__(self, name, description, method_name): + self._base_name = name + self._base_description = description + self._method_name = method_name + + def name(self): + return self._base_name + + def argsused(self): + return [ "nb_threads", "name" ] + + def description(self): + return self._base_description + + def run(self, perftools, args): + args['name'] = self._base_name + res = getattr(perftools, self._method_name)(self._base_name, nb_threads=args['nb_threads']); + print (res['pretty']) + + @staticmethod + def initTester(args): + os.environ["NSSLAPD_DB_LIB"] = args['db_lib'] + perftools = PerformanceTools( args ) + perftools.initInstance() + return perftools; + + class TesterImportExport(Tester): + # A special tester for export/import + def __init__(self): + super().__init__("export/import", + "Measure export rate in entries per seconds then measure import rate.", + None) + + def argsused(self): + return [] + + def run(self, perftools, args=None): + res = perftools.mesure_export_import() + for r in res: + print (r['pretty']) + + @staticmethod + def listTests(): + # List of test for which args.nb_threads is useful + return { t.name() : t for t in [ + PerformanceTools.Tester("search_uid", "Measure number of searches per seconds using filter with random existing uid.", "measure_search_by_uid"), + PerformanceTools.Tester("search_uid_in_dn", "Measure number of searches per seconds using filter with random existing uid in dn (i.e: (uid:dn:uid_value)).", "measure_search_by_filtering_the_dn"), + PerformanceTools.Tester("modify_sn", "Measure number of modify per seconds replacing sn by random value on random entries.", "measure_modify"), + PerformanceTools.TesterImportExport(), + ] } + + @staticmethod + def runAllTests(options): + for users in ( 100, 1000, 10000, 100000, 1000000 ): + for db in ( 'bdb', 'mdb' ): + perftools = PerformanceTools.Tester.initTester({**options, 'nbUsers': users, 'db_lib': db}) + for t in PerformanceTools.listTests().values(): + if 'nb_threads' in t.argsused(): + for nbthreads in ( 1, 4, 8 ): + t.run(perftools, { "nb_threads" : nbthreads }) + else: + t.run(perftools) + + diff --git a/src/lib389/lib389/topologies.py b/src/lib389/lib389/topologies.py deleted file mode 100644 index f4f2dbe10..000000000 --- a/src/lib389/lib389/topologies.py +++ /dev/null @@ -1,561 +0,0 @@ -# --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2021 Red Hat, Inc. -# All rights reserved. -# -# License: GPL (version 3 or any later version). -# See LICENSE for details. -# --- END COPYRIGHT BLOCK --- -# -import os -import logging -import socket # For hostname detection for GSSAPI tests -import pytest -from lib389 import DirSrv -from lib389.utils import generate_ds_params, is_fips -from lib389.mit_krb5 import MitKrb5 -from lib389.saslmap import SaslMappings -from lib389.replica import ReplicationManager, Replicas -from lib389.nss_ssl import NssSsl -from lib389._constants import * -from lib389.cli_base import LogCapture - -TLS_HOSTNAME_CHECK = os.getenv('TLS_HOSTNAME_CHECK', default=True) -DEBUGGING = os.getenv('DEBUGGING', default=False) -if DEBUGGING: - logging.getLogger(__name__).setLevel(logging.DEBUG) -else: - logging.getLogger(__name__).setLevel(logging.INFO) -log = logging.getLogger(__name__) - - -def _remove_ssca_db(topology): - ssca = NssSsl(dbpath=topology[0].get_ssca_dir()) - if ssca._db_exists(): - return ssca.remove_db() - else: - return True - - -def _create_instances(topo_dict, suffix): - """Create requested instances without replication or any other modifications - - :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.SUPPLIER: num, - ReplicaRole.HUB: num, ReplicaRole.CONSUMER: num} - :type topo_dict: dict - :param suffix: a suffix - :type suffix: str - - :return - TopologyMain object - """ - - instances = {} - ms = {} - cs = {} - hs = {} - ins = {} - - # Create instances - for role in topo_dict.keys(): - for inst_num in range(1, topo_dict[role]+1): - instance_data = generate_ds_params(inst_num, role) - if DEBUGGING: - instance = DirSrv(verbose=True) - else: - instance = DirSrv(verbose=False) - # TODO: Put 'args_instance' to generate_ds_params. - # Also, we need to keep in mind that the function returns - # SER_SECURE_PORT and REPLICA_ID that are not used in - # the instance creation here. - # args_instance[SER_HOST] = instance_data[SER_HOST] - args_instance = {} - args_instance[SER_PORT] = instance_data[SER_PORT] - args_instance[SER_SECURE_PORT] = instance_data[SER_SECURE_PORT] - args_instance[SER_SERVERID_PROP] = instance_data[SER_SERVERID_PROP] - # It's required to be able to make a suffix-less install for - # some cli tests. It's invalid to require replication with - # no suffix however .... - if suffix is not None: - args_instance[SER_CREATION_SUFFIX] = suffix - elif role != ReplicaRole.STANDALONE: - raise AssertionError("Invalid request to make suffix-less replicated environment") - - instance.allocate(args_instance) - - instance_exists = instance.exists() - - if instance_exists: - instance.delete() - - instance.create() - # We set a URL here to force ldap:// only. Once we turn on TLS - # we'll flick this to ldaps. - instance.use_ldap_uri() - instance.open() - instance.config.set('nsslapd-accesslog-logbuffering','off') - if role == ReplicaRole.STANDALONE: - ins[instance.serverid] = instance - instances.update(ins) - if role == ReplicaRole.SUPPLIER: - ms[instance.serverid] = instance - instances.update(ms) - if role == ReplicaRole.CONSUMER: - cs[instance.serverid] = instance - instances.update(cs) - if role == ReplicaRole.HUB: - hs[instance.serverid] = instance - instances.update(hs) - # We should always enable TLS while in FIPS mode because otherwise NSS database won't be - # configured in a FIPS compliant way - if is_fips(): - instance.enable_tls() - - # Disable strict hostname checking for TLS - if not TLS_HOSTNAME_CHECK: - instance.config.set('nsslapd-ssl-check-hostname', 'off') - if DEBUGGING: - instance.config.set('nsslapd-errorlog-level','8192') - instance.config.set('nsslapd-accesslog-level','260') - instance.config.set('nsslapd-auditlog-logging-enabled','on') - instance.config.set('nsslapd-auditfaillog-logging-enabled','on') - instance.config.set('nsslapd-plugin-logging', 'on') - log.info("Instance with parameters {} was created.".format(args_instance)) - - if "standalone1" in instances and len(instances) == 1: - return TopologyMain(standalones=instances["standalone1"]) - else: - return TopologyMain(standalones=ins, suppliers=ms, consumers=cs, hubs=hs) - - -def create_topology(topo_dict, suffix=DEFAULT_SUFFIX): - """Create a requested topology. Cascading replication scenario isn't supported - - :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.SUPPLIER: num, - ReplicaRole.CONSUMER: num} - :type topo_dict: dict - :param suffix: a suffix for the replication - :type suffix: str - - :return - TopologyMain object - """ - - if not topo_dict: - ValueError("You need to specify the dict. For instance: {ReplicaRole.STANDALONE: 1}") - - if ReplicaRole.HUB in topo_dict.keys(): - NotImplementedError("Cascading replication scenario isn't supported." - "Please, use existing topology or create your own.") - - topo = _create_instances(topo_dict, suffix) - - # Start with a single supplier, and create it "first". - first_supplier = None - try: - first_supplier = list(topo.ms.values())[0] - log.info("Creating replication topology.") - # Now get the first supplier ready. - repl = ReplicationManager(DEFAULT_SUFFIX) - repl.create_first_supplier(first_supplier) - except IndexError: - pass - - # Now init the other suppliers from this. - # This will reinit m, and put a bi-directional agreement - # in place. - for m in topo.ms.values(): - # Skip firstsupplier. - if m is first_supplier: - continue - log.info("Joining supplier %s to %s ..." % (m.serverid, first_supplier.serverid)) - repl.join_supplier(first_supplier, m) - - # Mesh the supplier agreements. - for mo in topo.ms.values(): - for mi in topo.ms.values(): - if mo is mi: - continue - log.info("Ensuring supplier %s to %s ..." % (mo.serverid, mi.serverid)) - repl.ensure_agreement(mo, mi) - - # Add supplier -> consumer agreements. - for c in topo.cs.values(): - log.info("Joining consumer %s from %s ..." % (c.serverid, first_supplier.serverid)) - repl.join_consumer(first_supplier, c) - - for m in topo.ms.values(): - for c in topo.cs.values(): - log.info("Ensuring consumer %s from %s ..." % (c.serverid, m.serverid)) - repl.ensure_agreement(m, c) - - # Clear out the tmp dir - for instance in topo: - instance.clearTmpDir(__file__) - - return topo - - -class TopologyMain(object): - def __init__(self, standalones=None, suppliers=None, consumers=None, hubs=None): - self.ms = {} - self.cs = {} - self.hs = {} - self.all_insts = {} - - if standalones: - if isinstance(standalones, dict): - self.ins = standalones - self.all_insts.update(standalones) - else: - self.standalone = standalones - self.all_insts['standalone1'] = standalones - if suppliers: - self.ms = suppliers - self.all_insts.update(self.ms) - if consumers: - self.cs = consumers - self.all_insts.update(self.cs) - if hubs: - self.hs = hubs - self.all_insts.update(self.hs) - - def __iter__(self): - return self.all_insts.values().__iter__() - - def __getitem__(self, index): - return list(self.all_insts.values())[index] - - def pause_all_replicas(self): - """Pause all agreements in the class instance""" - - for inst in self.all_insts.values(): - replicas = Replicas(inst) - replica = replicas.get(DEFAULT_SUFFIX) - for agreement in replica.get_agreements().list(): - agreement.pause() - - def resume_all_replicas(self): - """Resume all agreements in the class instance""" - - for inst in self.all_insts.values(): - replicas = Replicas(inst) - replica = replicas.get(DEFAULT_SUFFIX) - for agreement in replica.get_agreements().list(): - agreement.resume() - - def all_get_dsldapobject(self, dn, otype): - result = [] - for inst in self.all_insts.values(): - o = otype(inst, dn) - result.append(o) - return result - - -@pytest.fixture(scope="module") -def topology_st(request): - """Create DS standalone instance""" - - topology = create_topology({ReplicaRole.STANDALONE: 1}) - - def fin(): - topology.standalone.stop() - if DEBUGGING is None: - assert _remove_ssca_db(topology) - if topology.standalone.exists(): - topology.standalone.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -gssapi_ack = pytest.mark.skipif(not os.environ.get('GSSAPI_ACK', False), reason="GSSAPI tests may damage system configuration.") - -@pytest.fixture(scope="module") -def topology_st_gssapi(request): - """Create a DS standalone instance with GSSAPI enabled. - - This will alter the instance to remove the secure port, to allow - GSSAPI to function. - """ - hostname = socket.gethostname().split('.', 1) - - # Assert we have a domain setup in some kind. - assert len(hostname) == 2 - - REALM = hostname[1].upper() - - topology = create_topology({ReplicaRole.STANDALONE: 1}) - - # Fix the hostname. - topology.standalone.host = socket.gethostname() - - krb = MitKrb5(realm=REALM, debug=DEBUGGING) - - # Destroy existing realm. - if krb.check_realm(): - krb.destroy_realm() - krb.create_realm() - - # Now add krb to our instance. - krb.create_principal(principal='ldap/%s' % topology.standalone.host) - krb.create_keytab(principal='ldap/%s' % topology.standalone.host, keytab='/etc/krb5.keytab') - os.chown('/etc/krb5.keytab', topology.standalone.get_user_uid(), topology.standalone.get_group_gid()) - - # Add sasl mappings - saslmappings = SaslMappings(topology.standalone) - # First, purge all the default maps. - [m.delete() for m in saslmappings.list()] - # Now create a single map that works for our case. - saslmappings.create(properties={ - 'cn': 'suffix map', - # Don't add the realm due to a SASL bug - # 'nsSaslMapRegexString': '\\(.*\\)@%s' % self.realm, - 'nsSaslMapRegexString': '\\(.*\\)', - 'nsSaslMapBaseDNTemplate': topology.standalone.creation_suffix, - 'nsSaslMapFilterTemplate': '(uid=\\1)' - }) - topology.standalone.realm = krb - - topology.standalone.config.set('nsslapd-localhost', topology.standalone.host) - - topology.standalone.sslport = None - - topology.standalone.restart() - - topology.standalone.clearTmpDir(__file__) - - def fin(): - topology.standalone.stop() - if DEBUGGING is None: - assert _remove_ssca_db(topology) - if topology.standalone.exists(): - topology.standalone.delete() - krb.destroy_realm() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_no_sample(request): - """Create instance without sample entries to reproduce not initialised database""" - - topology = create_topology({ReplicaRole.STANDALONE: 1}, None) - topology.standalone.backends.create(properties={ - 'cn': 'userRoot', - 'nsslapd-suffix': DEFAULT_SUFFIX, - }) - - def fin(): - topology.standalone.stop() - if DEBUGGING is None: - assert _remove_ssca_db(topology) - if topology.standalone.exists(): - topology.standalone.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_i2(request): - """Create two instance DS deployment""" - - topology = create_topology({ReplicaRole.STANDALONE: 2}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_i3(request): - """Create three instance DS deployment""" - - topology = create_topology({ReplicaRole.STANDALONE: 3}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - -@pytest.fixture(scope="module") -def topology_m1(request): - """Create Replication Deployment with one supplier and one consumer""" - - topology = create_topology({ReplicaRole.SUPPLIER: 1}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - -@pytest.fixture(scope="module") -def topology_m1c1(request): - """Create Replication Deployment with one supplier and one consumer""" - - topology = create_topology({ReplicaRole.SUPPLIER: 1, - ReplicaRole.CONSUMER: 1}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_m2(request): - """Create Replication Deployment with two suppliers""" - - topology = create_topology({ReplicaRole.SUPPLIER: 2}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_m3(request): - """Create Replication Deployment with three suppliers""" - - topology = create_topology({ReplicaRole.SUPPLIER: 3}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_m4(request): - """Create Replication Deployment with four suppliers""" - - topology = create_topology({ReplicaRole.SUPPLIER: 4}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_m2c2(request): - """Create Replication Deployment with two suppliers and two consumers""" - - topology = create_topology({ReplicaRole.SUPPLIER: 2, - ReplicaRole.CONSUMER: 2}) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology - - -@pytest.fixture(scope="module") -def topology_m1h1c1(request): - """Create Replication Deployment with one supplier, one consumer and one hub""" - - topo_roles = {ReplicaRole.SUPPLIER: 1, ReplicaRole.HUB: 1, ReplicaRole.CONSUMER: 1} - topology = _create_instances(topo_roles, DEFAULT_SUFFIX) - supplier = topology.ms["supplier1"] - hub = topology.hs["hub1"] - consumer = topology.cs["consumer1"] - - # Start with the supplier, and create it "first". - log.info("Creating replication topology.") - # Now get the first supplier ready. - repl = ReplicationManager(DEFAULT_SUFFIX) - repl.create_first_supplier(supplier) - # Finish the topology creation - repl.join_hub(supplier, hub) - repl.join_consumer(hub, consumer) - - repl.test_replication(supplier, consumer) - - # Clear out the tmp dir - for instance in topology: - instance.clearTmpDir(__file__) - - def fin(): - [inst.stop() for inst in topology] - if DEBUGGING is None: - assert _remove_ssca_db(topology) - for inst in topology: - if inst.exists(): - inst.delete() - - request.addfinalizer(fin) - - topology.logcap = LogCapture() - return topology diff --git a/src/lib389/lib389/topologies/__init__.py b/src/lib389/lib389/topologies/__init__.py new file mode 100644 index 000000000..f4f2dbe10 --- /dev/null +++ b/src/lib389/lib389/topologies/__init__.py @@ -0,0 +1,561 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2021 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# +import os +import logging +import socket # For hostname detection for GSSAPI tests +import pytest +from lib389 import DirSrv +from lib389.utils import generate_ds_params, is_fips +from lib389.mit_krb5 import MitKrb5 +from lib389.saslmap import SaslMappings +from lib389.replica import ReplicationManager, Replicas +from lib389.nss_ssl import NssSsl +from lib389._constants import * +from lib389.cli_base import LogCapture + +TLS_HOSTNAME_CHECK = os.getenv('TLS_HOSTNAME_CHECK', default=True) +DEBUGGING = os.getenv('DEBUGGING', default=False) +if DEBUGGING: + logging.getLogger(__name__).setLevel(logging.DEBUG) +else: + logging.getLogger(__name__).setLevel(logging.INFO) +log = logging.getLogger(__name__) + + +def _remove_ssca_db(topology): + ssca = NssSsl(dbpath=topology[0].get_ssca_dir()) + if ssca._db_exists(): + return ssca.remove_db() + else: + return True + + +def _create_instances(topo_dict, suffix): + """Create requested instances without replication or any other modifications + + :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.SUPPLIER: num, + ReplicaRole.HUB: num, ReplicaRole.CONSUMER: num} + :type topo_dict: dict + :param suffix: a suffix + :type suffix: str + + :return - TopologyMain object + """ + + instances = {} + ms = {} + cs = {} + hs = {} + ins = {} + + # Create instances + for role in topo_dict.keys(): + for inst_num in range(1, topo_dict[role]+1): + instance_data = generate_ds_params(inst_num, role) + if DEBUGGING: + instance = DirSrv(verbose=True) + else: + instance = DirSrv(verbose=False) + # TODO: Put 'args_instance' to generate_ds_params. + # Also, we need to keep in mind that the function returns + # SER_SECURE_PORT and REPLICA_ID that are not used in + # the instance creation here. + # args_instance[SER_HOST] = instance_data[SER_HOST] + args_instance = {} + args_instance[SER_PORT] = instance_data[SER_PORT] + args_instance[SER_SECURE_PORT] = instance_data[SER_SECURE_PORT] + args_instance[SER_SERVERID_PROP] = instance_data[SER_SERVERID_PROP] + # It's required to be able to make a suffix-less install for + # some cli tests. It's invalid to require replication with + # no suffix however .... + if suffix is not None: + args_instance[SER_CREATION_SUFFIX] = suffix + elif role != ReplicaRole.STANDALONE: + raise AssertionError("Invalid request to make suffix-less replicated environment") + + instance.allocate(args_instance) + + instance_exists = instance.exists() + + if instance_exists: + instance.delete() + + instance.create() + # We set a URL here to force ldap:// only. Once we turn on TLS + # we'll flick this to ldaps. + instance.use_ldap_uri() + instance.open() + instance.config.set('nsslapd-accesslog-logbuffering','off') + if role == ReplicaRole.STANDALONE: + ins[instance.serverid] = instance + instances.update(ins) + if role == ReplicaRole.SUPPLIER: + ms[instance.serverid] = instance + instances.update(ms) + if role == ReplicaRole.CONSUMER: + cs[instance.serverid] = instance + instances.update(cs) + if role == ReplicaRole.HUB: + hs[instance.serverid] = instance + instances.update(hs) + # We should always enable TLS while in FIPS mode because otherwise NSS database won't be + # configured in a FIPS compliant way + if is_fips(): + instance.enable_tls() + + # Disable strict hostname checking for TLS + if not TLS_HOSTNAME_CHECK: + instance.config.set('nsslapd-ssl-check-hostname', 'off') + if DEBUGGING: + instance.config.set('nsslapd-errorlog-level','8192') + instance.config.set('nsslapd-accesslog-level','260') + instance.config.set('nsslapd-auditlog-logging-enabled','on') + instance.config.set('nsslapd-auditfaillog-logging-enabled','on') + instance.config.set('nsslapd-plugin-logging', 'on') + log.info("Instance with parameters {} was created.".format(args_instance)) + + if "standalone1" in instances and len(instances) == 1: + return TopologyMain(standalones=instances["standalone1"]) + else: + return TopologyMain(standalones=ins, suppliers=ms, consumers=cs, hubs=hs) + + +def create_topology(topo_dict, suffix=DEFAULT_SUFFIX): + """Create a requested topology. Cascading replication scenario isn't supported + + :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.SUPPLIER: num, + ReplicaRole.CONSUMER: num} + :type topo_dict: dict + :param suffix: a suffix for the replication + :type suffix: str + + :return - TopologyMain object + """ + + if not topo_dict: + ValueError("You need to specify the dict. For instance: {ReplicaRole.STANDALONE: 1}") + + if ReplicaRole.HUB in topo_dict.keys(): + NotImplementedError("Cascading replication scenario isn't supported." + "Please, use existing topology or create your own.") + + topo = _create_instances(topo_dict, suffix) + + # Start with a single supplier, and create it "first". + first_supplier = None + try: + first_supplier = list(topo.ms.values())[0] + log.info("Creating replication topology.") + # Now get the first supplier ready. + repl = ReplicationManager(DEFAULT_SUFFIX) + repl.create_first_supplier(first_supplier) + except IndexError: + pass + + # Now init the other suppliers from this. + # This will reinit m, and put a bi-directional agreement + # in place. + for m in topo.ms.values(): + # Skip firstsupplier. + if m is first_supplier: + continue + log.info("Joining supplier %s to %s ..." % (m.serverid, first_supplier.serverid)) + repl.join_supplier(first_supplier, m) + + # Mesh the supplier agreements. + for mo in topo.ms.values(): + for mi in topo.ms.values(): + if mo is mi: + continue + log.info("Ensuring supplier %s to %s ..." % (mo.serverid, mi.serverid)) + repl.ensure_agreement(mo, mi) + + # Add supplier -> consumer agreements. + for c in topo.cs.values(): + log.info("Joining consumer %s from %s ..." % (c.serverid, first_supplier.serverid)) + repl.join_consumer(first_supplier, c) + + for m in topo.ms.values(): + for c in topo.cs.values(): + log.info("Ensuring consumer %s from %s ..." % (c.serverid, m.serverid)) + repl.ensure_agreement(m, c) + + # Clear out the tmp dir + for instance in topo: + instance.clearTmpDir(__file__) + + return topo + + +class TopologyMain(object): + def __init__(self, standalones=None, suppliers=None, consumers=None, hubs=None): + self.ms = {} + self.cs = {} + self.hs = {} + self.all_insts = {} + + if standalones: + if isinstance(standalones, dict): + self.ins = standalones + self.all_insts.update(standalones) + else: + self.standalone = standalones + self.all_insts['standalone1'] = standalones + if suppliers: + self.ms = suppliers + self.all_insts.update(self.ms) + if consumers: + self.cs = consumers + self.all_insts.update(self.cs) + if hubs: + self.hs = hubs + self.all_insts.update(self.hs) + + def __iter__(self): + return self.all_insts.values().__iter__() + + def __getitem__(self, index): + return list(self.all_insts.values())[index] + + def pause_all_replicas(self): + """Pause all agreements in the class instance""" + + for inst in self.all_insts.values(): + replicas = Replicas(inst) + replica = replicas.get(DEFAULT_SUFFIX) + for agreement in replica.get_agreements().list(): + agreement.pause() + + def resume_all_replicas(self): + """Resume all agreements in the class instance""" + + for inst in self.all_insts.values(): + replicas = Replicas(inst) + replica = replicas.get(DEFAULT_SUFFIX) + for agreement in replica.get_agreements().list(): + agreement.resume() + + def all_get_dsldapobject(self, dn, otype): + result = [] + for inst in self.all_insts.values(): + o = otype(inst, dn) + result.append(o) + return result + + +@pytest.fixture(scope="module") +def topology_st(request): + """Create DS standalone instance""" + + topology = create_topology({ReplicaRole.STANDALONE: 1}) + + def fin(): + topology.standalone.stop() + if DEBUGGING is None: + assert _remove_ssca_db(topology) + if topology.standalone.exists(): + topology.standalone.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +gssapi_ack = pytest.mark.skipif(not os.environ.get('GSSAPI_ACK', False), reason="GSSAPI tests may damage system configuration.") + +@pytest.fixture(scope="module") +def topology_st_gssapi(request): + """Create a DS standalone instance with GSSAPI enabled. + + This will alter the instance to remove the secure port, to allow + GSSAPI to function. + """ + hostname = socket.gethostname().split('.', 1) + + # Assert we have a domain setup in some kind. + assert len(hostname) == 2 + + REALM = hostname[1].upper() + + topology = create_topology({ReplicaRole.STANDALONE: 1}) + + # Fix the hostname. + topology.standalone.host = socket.gethostname() + + krb = MitKrb5(realm=REALM, debug=DEBUGGING) + + # Destroy existing realm. + if krb.check_realm(): + krb.destroy_realm() + krb.create_realm() + + # Now add krb to our instance. + krb.create_principal(principal='ldap/%s' % topology.standalone.host) + krb.create_keytab(principal='ldap/%s' % topology.standalone.host, keytab='/etc/krb5.keytab') + os.chown('/etc/krb5.keytab', topology.standalone.get_user_uid(), topology.standalone.get_group_gid()) + + # Add sasl mappings + saslmappings = SaslMappings(topology.standalone) + # First, purge all the default maps. + [m.delete() for m in saslmappings.list()] + # Now create a single map that works for our case. + saslmappings.create(properties={ + 'cn': 'suffix map', + # Don't add the realm due to a SASL bug + # 'nsSaslMapRegexString': '\\(.*\\)@%s' % self.realm, + 'nsSaslMapRegexString': '\\(.*\\)', + 'nsSaslMapBaseDNTemplate': topology.standalone.creation_suffix, + 'nsSaslMapFilterTemplate': '(uid=\\1)' + }) + topology.standalone.realm = krb + + topology.standalone.config.set('nsslapd-localhost', topology.standalone.host) + + topology.standalone.sslport = None + + topology.standalone.restart() + + topology.standalone.clearTmpDir(__file__) + + def fin(): + topology.standalone.stop() + if DEBUGGING is None: + assert _remove_ssca_db(topology) + if topology.standalone.exists(): + topology.standalone.delete() + krb.destroy_realm() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_no_sample(request): + """Create instance without sample entries to reproduce not initialised database""" + + topology = create_topology({ReplicaRole.STANDALONE: 1}, None) + topology.standalone.backends.create(properties={ + 'cn': 'userRoot', + 'nsslapd-suffix': DEFAULT_SUFFIX, + }) + + def fin(): + topology.standalone.stop() + if DEBUGGING is None: + assert _remove_ssca_db(topology) + if topology.standalone.exists(): + topology.standalone.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_i2(request): + """Create two instance DS deployment""" + + topology = create_topology({ReplicaRole.STANDALONE: 2}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_i3(request): + """Create three instance DS deployment""" + + topology = create_topology({ReplicaRole.STANDALONE: 3}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + +@pytest.fixture(scope="module") +def topology_m1(request): + """Create Replication Deployment with one supplier and one consumer""" + + topology = create_topology({ReplicaRole.SUPPLIER: 1}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + +@pytest.fixture(scope="module") +def topology_m1c1(request): + """Create Replication Deployment with one supplier and one consumer""" + + topology = create_topology({ReplicaRole.SUPPLIER: 1, + ReplicaRole.CONSUMER: 1}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_m2(request): + """Create Replication Deployment with two suppliers""" + + topology = create_topology({ReplicaRole.SUPPLIER: 2}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_m3(request): + """Create Replication Deployment with three suppliers""" + + topology = create_topology({ReplicaRole.SUPPLIER: 3}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_m4(request): + """Create Replication Deployment with four suppliers""" + + topology = create_topology({ReplicaRole.SUPPLIER: 4}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_m2c2(request): + """Create Replication Deployment with two suppliers and two consumers""" + + topology = create_topology({ReplicaRole.SUPPLIER: 2, + ReplicaRole.CONSUMER: 2}) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology + + +@pytest.fixture(scope="module") +def topology_m1h1c1(request): + """Create Replication Deployment with one supplier, one consumer and one hub""" + + topo_roles = {ReplicaRole.SUPPLIER: 1, ReplicaRole.HUB: 1, ReplicaRole.CONSUMER: 1} + topology = _create_instances(topo_roles, DEFAULT_SUFFIX) + supplier = topology.ms["supplier1"] + hub = topology.hs["hub1"] + consumer = topology.cs["consumer1"] + + # Start with the supplier, and create it "first". + log.info("Creating replication topology.") + # Now get the first supplier ready. + repl = ReplicationManager(DEFAULT_SUFFIX) + repl.create_first_supplier(supplier) + # Finish the topology creation + repl.join_hub(supplier, hub) + repl.join_consumer(hub, consumer) + + repl.test_replication(supplier, consumer) + + # Clear out the tmp dir + for instance in topology: + instance.clearTmpDir(__file__) + + def fin(): + [inst.stop() for inst in topology] + if DEBUGGING is None: + assert _remove_ssca_db(topology) + for inst in topology: + if inst.exists(): + inst.delete() + + request.addfinalizer(fin) + + topology.logcap = LogCapture() + return topology diff --git a/src/lib389/setup.py b/src/lib389/setup.py index ebfbf636b..3ad6f2810 100644 --- a/src/lib389/setup.py +++ b/src/lib389/setup.py @@ -68,7 +68,16 @@ setup( 'Topic :: Software Development :: Testing'], keywords='389 directory server test configure', - packages=find_packages(exclude=['tests*']), + packages=find_packages( + exclude=[ + '*.tests', + '*.tests.*', + 'lib389.topologies', + 'lib389.topologies.*', + 'lib389.perftools', + 'lib389.perftools.*', + ] + ), # find lib389/clitools -name ds\* -exec echo \''{}'\', \; data_files=[