diff --git a/README.md b/README.md index 0e80cc8..de92a89 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ After benchmarking has been finished we will see a message like Experiment has been finished ``` -The script has created a result folder in the current directory containing the results. `` is the name of the folder. +The script has created a result folder in the current directory containing the results. `code` is the name of the folder. ### Evaluate Results in Dashboard @@ -122,7 +122,7 @@ The script has created a result folder in the current directory containing the r Run the command: `dbmsdashboard` This will start the evaluation dashboard at `localhost:8050`. -Visit the address in a browser and select the experiment ``. +Visit the address in a browser and select the experiment `code`. Alternatively you may use a [Jupyter notebook](https://github.com/Beuth-Erdelt/DBMS-Benchmarker/blob/master/Evaluation-Demo.ipynb), see a [rendered example](https://beuth-erdelt.github.io/DBMS-Benchmarker/Evaluation-Demo.html). diff --git a/dbmsbenchmarker/benchmarker.py b/dbmsbenchmarker/benchmarker.py index 38e04f4..5cbb138 100644 --- a/dbmsbenchmarker/benchmarker.py +++ b/dbmsbenchmarker/benchmarker.py @@ -2032,7 +2032,7 @@ def run_cli(parameter): result_folder = './' else: result_folder = args.result_folder - command_args = vars(args) + command_args = vars(args).copy() makedirs(result_folder+"/"+code) copyfile(args.config_folder+'/connections.config', result_folder+"/"+code+'/connections.config')#args.connection_file) copyfile(args.config_folder+'/queries.config', result_folder+"/"+code+'/queries.config')#args.query_file) @@ -2058,10 +2058,14 @@ def run_cli(parameter): command_args['copy_subfolder'] = True command_args['subfolder'] = connection command_args['connection'] = connection + #if 'generate_evaluation' in command_args: + # del command_args['generate_evaluation'] + command_args['generate_evaluation'] = 'no' # Get the current UTC time current_time = datetime.datetime.utcnow() # Add 5 seconds to the current time - start_time = current_time + datetime.timedelta(seconds=5) + seconds = 5 if 5 > numProcesses else numProcesses + start_time = current_time + datetime.timedelta(seconds=seconds) command_args['start_time'] = start_time.strftime('%Y-%m-%d %H:%M:%S') #command_args['stream_id'] = 1 pool_args = []#(dict(command_args),)]*numProcesses @@ -2098,7 +2102,7 @@ def run_cli(parameter): #command_args['result_folder'] = code #experiments = benchmarker.run_cli(command_args) experiments = benchmarker( - result_path=args.result_folder, + result_path=result_folder,#args.result_folder, code=code, #working=args.working, batch=bBatch, @@ -2118,7 +2122,7 @@ def run_cli(parameter): experiments.getConfig() experiments.readBenchmarks() evaluate = run_evaluation(experiments) - print("Experiment {} has been finished".format(experiments.code)) + #print("Experiment {} has been finished".format(experiments.code)) #print(evaluate) #list_connections = evaluate.get_experiment_list_connections() #print(list_connections) @@ -2134,7 +2138,8 @@ def run_cli(parameter): time_end = max(times_end) print(time_start, time_end, time_end-time_start) """ - return experiments + return experiments + return None else: if args.mode != 'read': # sleep before going to work @@ -2286,6 +2291,19 @@ def run_evaluation(experiments): #num_processes = min(float(args.numProcesses if not args.numProcesses is None else 1), float(args.num_run) if int(args.num_run) > 0 else 1) evaluate = dbmsbenchmarker.inspector.inspector(result_folder) evaluate.load_experiment("")#experiments.code) + print("Show Evaluation") + print("===============") + # get workload properties + workload_properties = evaluate.get_experiment_workload_properties() + query_properties = evaluate.get_experiment_query_properties() + def map_index_to_queryname(numQuery): + if numQuery[1:] in query_properties and 'config' in query_properties[numQuery[1:]] and 'title' in query_properties[numQuery[1:]]['config']: + return query_properties[numQuery[1:]]['config']['title'] + else: + return numQuery + #query_properties['1']['config'] + print(workload_properties['name'], ":", workload_properties['intro']) + # get queries and dbms list_queries_all = evaluate.get_experiment_list_queries() #print(list_queries_all) dbms_filter = [] @@ -2295,11 +2313,11 @@ def run_evaluation(experiments): df = evaluate.get_timer(q, "execution") if len(list(df.index)) > 0: dbms_filter = list(df.index) - print("First successful query: {}".format(q)) + print("First successful query: Q{}".format(q)) break #print(dbms_filter) #list_queries = evaluate.get_experiment_queries_successful() # evaluate.get_experiment_list_queries() - list_queries = evaluate.get_survey_successful(timername='execution', dbms_filter=dbms_filter) + list_queries = evaluate.get_survey_successful(timername='run', dbms_filter=dbms_filter) #print(list_queries, len(list_queries)) if 'numRun' in experiments.connectionmanagement: num_run = experiments.connectionmanagement['numRun'] @@ -2317,16 +2335,28 @@ def run_evaluation(experiments): print("Number of max. parallel clients:", int(num_processes)) ##################### print("\n### Errors (failed queries)") - print(evaluate.get_total_errors(dbms_filter=dbms_filter).T) + df = evaluate.get_total_errors(dbms_filter=dbms_filter).T + num_errors = df.sum().sum() + if num_errors > 0: + df.index = df.index.map(map_index_to_queryname) + print(df) + else: + print("No errors") ##################### print("\n### Warnings (result mismatch)") - print(evaluate.get_total_warnings(dbms_filter=dbms_filter).T) + df = evaluate.get_total_warnings(dbms_filter=dbms_filter).T + num_warnings = df.sum().sum() + if num_warnings > 0: + df.index = df.index.map(map_index_to_queryname) + print(df) + else: + print("No warnings") ##################### #df = evaluate.get_aggregated_query_statistics(type='timer', name='connection', query_aggregate='Median', dbms_filter=dbms_filter) df = evaluate.get_aggregated_experiment_statistics(type='timer', name='connection', query_aggregate='Median', total_aggregate='Geo', dbms_filter=dbms_filter) df = (df/1000.0).sort_index() if not df.empty: - print("### Geometric Mean of Medians of Connection Times (only successful) [s]") + print("\n### Geometric Mean of Medians of Connection Times (only successful) [s]") df.columns = ['average connection time [s]'] print(df.round(2)) #print("### Statistics of Timer Connection (only successful) [s]") @@ -2337,7 +2367,7 @@ def run_evaluation(experiments): df = evaluate.get_aggregated_experiment_statistics(type='timer', name='connection', query_aggregate='Max', total_aggregate='Max', dbms_filter=dbms_filter) df = (df/1000.0).sort_index() if not df.empty: - print("### Max of Connection Times (only successful) [s]") + print("\n### Max of Connection Times (only successful) [s]") df.columns = ['max connection time [s]'] print(df.round(2)) #print("### Statistics of Timer Connection (only successful) [s]") @@ -2347,7 +2377,7 @@ def run_evaluation(experiments): df = evaluate.get_aggregated_experiment_statistics(type='timer', name='run', query_aggregate='Median', total_aggregate='Geo', dbms_filter=dbms_filter) df = (df/1000.0).sort_index() if not df.empty: - print("### Geometric Mean of Medians of Run Times (only successful) [s]") + print("\n### Geometric Mean of Medians of Run Times (only successful) [s]") df.columns = ['average run time [s]'] print(df.round(2)) ##################### @@ -2432,6 +2462,7 @@ def run_evaluation(experiments): df = pd.DataFrame.from_dict(tpx_total, orient='index')#, columns=['queries per hour [Qph]']) df.index.name = 'DBMS' print(df) + print("Experiment {} has been finished".format(experiments.code)) return evaluate diff --git a/dbmsbenchmarker/inspector.py b/dbmsbenchmarker/inspector.py index 9ca3982..ace3299 100644 --- a/dbmsbenchmarker/inspector.py +++ b/dbmsbenchmarker/inspector.py @@ -544,7 +544,11 @@ def get_experiment_queries_successful(self, dbms_filter=[]): def get_survey_successful(self, timername=None, dbms_filter=[]): # list of active queries for timer e[0] = execution if not timername is None: + #print(self.benchmarks.timers) + #timers = [(i, t.name) for i,t in enumerate(self.benchmarks.timers)] + #print(timers) epos = [i for i,t in enumerate(self.benchmarks.timers) if t.name==timername] + #print(epos, self.get_experiment_queries_successful(dbms_filter=dbms_filter)) l = self.get_experiment_queries_successful(dbms_filter=dbms_filter)[epos[0]] return l else: diff --git a/dbmsbenchmarker/tools.py b/dbmsbenchmarker/tools.py index 2a113b1..90de7bd 100644 --- a/dbmsbenchmarker/tools.py +++ b/dbmsbenchmarker/tools.py @@ -27,6 +27,7 @@ from os import path import matplotlib.pyplot as plt import pickle +import traceback from dbmsbenchmarker import inspector @@ -1278,7 +1279,8 @@ def evaluateResultsizeToDataFrame(evaluation): d = df.replace(0, np.nan).min() df = df.T df.index = ['Q'+j for i,j in enumerate(df.index)] - df.columns = list(l["1"].keys()) + # find first active query + df.columns = list(l[list(l.keys())[0]].keys()) df.columns = df.columns.map(dbms.anonymizer) df = df.reindex(sorted(df.columns), axis=1) return df @@ -1292,7 +1294,8 @@ def evaluateNormalizedResultsizeToDataFrame(evaluation): df = df.div(d).replace(0,np.nan) df = df.T df.index = ['Q'+j for i,j in enumerate(df.index)] - df.columns = list(l["1"].keys()) + # find first active query + df.columns = list(l[list(l.keys())[0]].keys()) df.columns = df.columns.map(dbms.anonymizer) df = df.reindex(sorted(df.columns), axis=1) return df @@ -1302,7 +1305,8 @@ def evaluateErrorsToDataFrame(evaluation): df = pd.DataFrame(l) df = df.T df.index = ['Q'+j for i,j in enumerate(df.index)] - df.columns = list(l["1"].keys()) + # find first active query + df.columns = list(l[list(l.keys())[0]].keys()) df.columns = df.columns.map(dbms.anonymizer) df = df.reindex(sorted(df.columns), axis=1) return df @@ -1312,7 +1316,8 @@ def evaluateWarningsToDataFrame(evaluation): df = pd.DataFrame(l) df = df.T df.index = ['Q'+j for i,j in enumerate(df.index)] - df.columns = list(l["1"].keys()) + # find first active query + df.columns = list(l[list(l.keys())[0]].keys()) df.columns = df.columns.map(dbms.anonymizer) df = df.reindex(sorted(df.columns), axis=1) return df @@ -1434,6 +1439,7 @@ def findSuccessfulQueriesAllDBMS(benchmarker, numQuery, timer, dbms_filter=[]): if not bIgnoreQuery: # no active dbms missing for this timer and query validQueries[numTimer].append(i) + logging.debug("Query is successful: {}".format(i)) return validQueries @@ -1647,6 +1653,7 @@ def joinDicts(d1, d2): data_first = None df_first = None connection_first = None + titles_result = [] for connection in list_connections: try: filename = '{folder}/{connection}/query_{numQuery}_resultset_complete_{connection}.pickle'.format(folder=folder, connection=connection, numQuery=numQuery) @@ -1659,7 +1666,9 @@ def joinDicts(d1, d2): if data_first is None: protocol['query'][numQuery]['dataStorage'] = data.copy() protocol['query'][numQuery]['warnings'][connection] = '' - titles_result = protocol['query'][numQuery]['dataStorage'][0][0] + if len(protocol['query'][numQuery]['dataStorage'][0]) > 0: + titles_result = protocol['query'][numQuery]['dataStorage'][0][0] + #print(protocol['query'][numQuery]['dataStorage'][0]) data_first = data.copy() connection_first = connection else: @@ -1668,7 +1677,10 @@ def joinDicts(d1, d2): #print("numRun {}".format(numRun), end='') result = data[numRun].copy() # remove titles - titles_result = data[numRun][0]#list(range(len(result[0]))) + if len(data[numRun]) > 0: + titles_result = data[numRun][0]#list(range(len(result[0]))) + else: + continue #print(titles_result) result.pop(0) # convert datatypes @@ -1731,33 +1743,35 @@ def joinDicts(d1, d2): # result set of first run only filename = '{folder}/{connection}/query_{numQuery}_resultset_{connection}.pickle'.format(folder=folder, connection=connection, numQuery=numQuery) #print(connection+": ", end='')#, df) - with open(filename, 'r') as f: - df = pd.read_pickle(filename) - #print(connection)#, df) - if df_first is None: - df_first = df.copy() - #print("first\n", df_first) - result_as_list = [[i[0] for i in list(df_first.columns)]] - result_as_list.extend(df_first.values.tolist()) - protocol['query'][numQuery]['dataStorage'] = [result_as_list] # list, because this is (only) first run - protocol['query'][numQuery]['warnings'][connection] = "" - else: - df_1 = inspector.getDifference12(df_first, df) - df_2 = inspector.getDifference12(df, df_first) - if not df_1.empty or not df_2.empty: - #print("different\n", df) - protocol['query'][numQuery]['warnings'][connection] = 'Different' - result_as_list = [[i[0] for i in list(df.columns)]] - result_as_list.extend(df.values.tolist()) - protocol['query'][numQuery]['resultSets'][connection] = [result_as_list] # list, because this is (only) first run - else: - #print("OK") - protocol['query'][numQuery]['resultSets'][connection] = [] + if isfile(filename): + with open(filename, 'r') as f: + df = pd.read_pickle(filename) + #print(connection)#, df) + if df_first is None: + df_first = df.copy() + #print("first\n", df_first) + result_as_list = [[i[0] for i in list(df_first.columns)]] + result_as_list.extend(df_first.values.tolist()) + protocol['query'][numQuery]['dataStorage'] = [result_as_list] # list, because this is (only) first run protocol['query'][numQuery]['warnings'][connection] = "" + else: + df_1 = inspector.getDifference12(df_first, df) + df_2 = inspector.getDifference12(df, df_first) + if not df_1.empty or not df_2.empty: + #print("different\n", df) + protocol['query'][numQuery]['warnings'][connection] = 'Different' + result_as_list = [[i[0] for i in list(df.columns)]] + result_as_list.extend(df.values.tolist()) + protocol['query'][numQuery]['resultSets'][connection] = [result_as_list] # list, because this is (only) first run + else: + #print("OK") + protocol['query'][numQuery]['resultSets'][connection] = [] + protocol['query'][numQuery]['warnings'][connection] = "" except Exception as e: print(e) #print("missing") protocol['query'][numQuery]['warnings'][connection] = 'Missing' + traceback.print_exc() finally: pass #print("warnings", protocol['query']['3']['warnings']) diff --git a/docs/DBMS.md b/docs/DBMS.md index 5736ed0..ecdabef 100644 --- a/docs/DBMS.md +++ b/docs/DBMS.md @@ -204,11 +204,12 @@ JDBC driver: https://www.monetdb.org/downloads/Java/ 'name': 'MonetDB', 'info': 'This is a demo of MonetDB', 'active': True, + 'dialect': 'MonetDB', 'JDBC': { - 'driver': 'nl.cwi.monetdb.jdbc.MonetDriver', - 'url': 'jdbc:monetdb://localhost:50000/database?so_timeout=10000', - 'auth': ['user', 'password'], - 'jar': 'jars/monetdb-jdbc-2.29.jar', + 'driver': 'org.monetdb.jdbc.MonetDriver', + 'url': 'jdbc:monetdb://localhost:50000/monetdb?so_timeout=10000', + 'auth': ['monetdb', 'monetdb'], + 'jar': 'jars/monetdb-jdbc-3.3.jre8', } }, ] diff --git a/docs/Example-TPC-H.md b/docs/Example-TPC-H.md index d64539c..cb07b2a 100644 --- a/docs/Example-TPC-H.md +++ b/docs/Example-TPC-H.md @@ -23,7 +23,7 @@ We need If necessary, adjust the settings in the file `example/tpc-h/connections.py`: -``` +```bash [ { 'name': 'PostgreSQL', @@ -64,7 +64,7 @@ The script has created a result folder in the current directory containing the r If the `-e yes` option is used, you will see something like -``` +```bash First successful query: 1 Limited to: ['PostgreSQL'] Number of runs per query: 1 @@ -146,10 +146,11 @@ PostgreSQL 9900.0 1*1*22*3600/8 Run the command: -`python dashboard.py` +`dbmsdashboard` This will start the evaluation dashboard at `localhost:8050`. -Visit the address in a browser and select the experiment ``. +Visit the address in a browser and select the experiment `code`. + ## Where to go diff --git a/requirements.txt b/requirements.txt index c50788d..37f1775 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,3 +45,4 @@ pillow>=10.0.1 # not directly required, pinned by Snyk to avoid a vulnerability setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability fonttools>=4.43.0 # not directly required, pinned by Snyk to avoid a vulnerability jinja2>=3.1.3 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/setup.py b/setup.py index fdadf90..19bb8c7 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="dbmsbenchmarker", - version="0.14.0", + version="0.14.1", author="Patrick Erdelt", author_email="perdelt@beuth-hochschule.de", description="DBMS-Benchmarker is a Python-based application-level blackbox benchmark tool for Database Management Systems (DBMS). It connects to a given list of DBMS (via JDBC) and runs a given list of parametrized and randomized (SQL) benchmark queries. Evaluations are available via Python interface, in reports and at an interactive multi-dimensional dashboard.",