Introduce silent mode (#171)

* Evaluation: Only finish message at the end of evaluation

* TPC-H: Current output

* fix: requirements.txt to reduce vulnerabilities

* Benchmarker: Bugfix verbosity

* Benchmarker:  nan instead of NaN for NumPy 2.0

* Evaluation: natural_sort results

* Evaluation: Catch error if time span is below 1 sec

* Benchmarker: fixed query works again even if not fixed to first

* Evaluation: Bugfix, if -pp derives names

* Evaluation: Bugfix, show correct number of streams

* Evaluation: Bugfix, show total throughput only if above 0 (i.e., successful execution exists)

* Benchmarker: silent mode (-vn)

* Benchmarker: better exception messages

* Benchmarker: new convert_to_rounded_float() to avoid SyntaxError: invalid decimal literal

* Benchmarker: Typo CLI help

* Benchmarker: new convert_to_rounded_float() to avoid SyntaxError: invalid decimal literal

* Benchmarker: Bugfix several streams with single connection, shuffling queries

* Benchmarker: Random seed is used before each query, not only before each stream

* Bump version to 0.14.2

benchmark.py

@@ -59,7 +59,8 @@
     parser.add_argument('-vq', '--verbose-queries', help='print every query that is sent', action='store_true', default=False)
     parser.add_argument('-vs', '--verbose-statistics', help='print statistics about queries that have been sent', action='store_true', default=False)
     parser.add_argument('-vr', '--verbose-results', help='print result sets of every query that has been sent', action='store_true', default=False)
-    parser.add_argument('-vp', '--verbose-process', help='print result sets of every query that has been sent', action='store_true', default=False)
+    parser.add_argument('-vp', '--verbose-process', help='print infos about the workflow steps', action='store_true', default=False)
+    parser.add_argument('-vn', '--verbose-none', help='stay completely silent', action='store_true', default=False)
     parser.add_argument('-pn', '--num-run', help='Parameter: Number of executions per query', default=0)
     parser.add_argument('-m', '--metrics', help='collect hardware metrics per query', action='store_true', default=False)
     parser.add_argument('-mps', '--metrics-per-stream', help='collect hardware metrics per stream', action='store_true', default=False)

dbmsbenchmarker/evaluator.py

@@ -486,7 +486,7 @@ def dfLatQ(query, warmup=0, cooldown=0):
 def addStatistics(dataframe, drop_nan=True, drop_measures=False):
     df = dataframe.copy().T
     # treat 0 as missing value
-    df = df.replace(0., np.NaN)
+    df = df.replace(0., np.nan)
     if drop_nan and df.isnull().any().any():
         #print("Missing!")
         with_nan = True
@@ -509,7 +509,7 @@ def addStatistics(dataframe, drop_nan=True, drop_measures=False):
     #stat_geo = np.exp(np.log(df.prod(axis=0))/df.notna().sum(1))
     #print("Geo", df)
     # remove 0 and nan
-    stat_geo = stats.gmean(df.replace(0., np.NaN).dropna(), axis=0)
+    stat_geo = stats.gmean(df.replace(0., np.nan).dropna(), axis=0)
     stat_n = df.count(axis=0).array
     #df.loc['n']= len(df.index)
     df.loc['n']= stat_n

dbmsbenchmarker/monitor.py

@@ -522,7 +522,7 @@ def clean_dataframe(dataframe):
     # helps evaluating GPU util
     # removes leading zeros in each row
     # shifts values > 0 to the beginning
-    # appends NaN to keep numbers of columns the same
+    # appends nan to keep numbers of columns the same
     # replaces all 0 by nan
     #print(dataframe)
     for rows in range(len(dataframe.index)):

dbmsbenchmarker/reporter.py

@@ -24,7 +24,7 @@
 import os
 import csv
 import json
-from dbmsbenchmarker import tools, monitor, evaluator
+from dbmsbenchmarker import tools, monitor, evaluator, benchmarker
 #import datetime
 from datetime import datetime
 from tqdm import tqdm
@@ -249,17 +249,18 @@ def generate(self, numQuery, timer):
                         l.extend(*times_output[c])
                     data.append(l)
         # print table
-        print('Q'+str(numQuery)+': '+query.title+" - timerExecution")
-        print(tabulate(data,headers=header, tablefmt="grid", floatfmt=".2f"))
-        # print errors
-        for key, value in self.benchmarker.protocol['query'][str(numQuery)]['errors'].items():
-            if key in self.benchmarker.dbms and self.benchmarker.dbms[key].connectiondata['active']:
-                if len(value) > 0:
-                    firstpos = value.find(': ')
-                    if firstpos > 0:
-                        print(key + ': ' + value[firstpos:])
-                    else:
-                        print(key + ': ' + value)
+        if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+            print('Q'+str(numQuery)+': '+query.title+" - timerExecution")
+            print(tabulate(data,headers=header, tablefmt="grid", floatfmt=".2f"))
+            # print errors
+            for key, value in self.benchmarker.protocol['query'][str(numQuery)]['errors'].items():
+                if key in self.benchmarker.dbms and self.benchmarker.dbms[key].connectiondata['active']:
+                    if len(value) > 0:
+                        firstpos = value.find(': ')
+                        if firstpos > 0:
+                            print(key + ': ' + value[firstpos:])
+                        else:
+                            print(key + ': ' + value)
         return data
 
 

dbmsbenchmarker/scripts/cli.py

@@ -55,7 +55,8 @@ def run_benchmarker():
     parser.add_argument('-vq', '--verbose-queries', help='print every query that is sent', action='store_true', default=False)
     parser.add_argument('-vs', '--verbose-statistics', help='print statistics about queries that have been sent', action='store_true', default=False)
     parser.add_argument('-vr', '--verbose-results', help='print result sets of every query that has been sent', action='store_true', default=False)
-    parser.add_argument('-vp', '--verbose-process', help='print result sets of every query that has been sent', action='store_true', default=False)
+    parser.add_argument('-vp', '--verbose-process', help='print infos about the workflow steps', action='store_true', default=False)
+    parser.add_argument('-vn', '--verbose-none', help='stay completely silent', action='store_true', default=False)
     parser.add_argument('-pn', '--num-run', help='Parameter: Number of executions per query', default=0)
     parser.add_argument('-m', '--metrics', help='collect hardware metrics per query', action='store_true', default=False)
     parser.add_argument('-mps', '--metrics-per-stream', help='collect hardware metrics per stream', action='store_true', default=False)

dbmsbenchmarker/tools.py

@@ -28,8 +28,9 @@
 import matplotlib.pyplot as plt
 import pickle
 import traceback
+import warnings
 
-from dbmsbenchmarker import inspector
+from dbmsbenchmarker import inspector, benchmarker
 
 # Set query timeout
 jaydebeapi.QUERY_TIMEOUT = 0
@@ -183,7 +184,7 @@ def skipTimer(self, numQuery, query, nameConnection):
         """
         self.nameConnection = nameConnection
         self.startTimerQuery(numQuery, query)#numWarmup, numRun)
-        if len(self.times) <= self.currentQuery:
+        while len(self.times) <= self.currentQuery:
             self.times.append({})
             self.stats.append({})
         self.finishTimerQuery()
@@ -624,9 +625,10 @@ def connect(self):
                 self.connectiondata['version'] = self.version
                 self.connectiondata['driver'] = self.driver
                 self.connectiondata['driverversion'] = self.driverversion
-                print("Connected to {} version {} using {} version {}".format(self.product, self.version, self.driver, self.driverversion))
+                if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+                    print("Connected to {} version {} using {} version {}".format(self.product, self.version, self.driver, self.driverversion))
             except Exception as e:
-                print("Product and version not implemented in JDBC driver")
+                print("Product and version not implemented in JDBC driver {}".format(self.connectiondata['JDBC']['jar']))
             else:
                 pass
             if 'init_SQL' in self.connectiondata:
@@ -1484,6 +1486,47 @@ def convertToInt(var):
         return var
 
 
+def convert_to_rounded_float(var, decimals=2):
+    """
+    Converts a variable to a rounded float if possible, otherwise returns the original value.
+
+    :param var: The variable to be converted.
+    :param decimals: The number of decimal places to round to.
+    :return: The rounded float or the original value if conversion is not possible.
+    """
+    def safe_literal_eval(var):
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always", SyntaxWarning)  # Catch all SyntaxWarnings
+            try:
+                result = ast.literal_eval(var)
+                # Check if a SyntaxWarning was issued
+                if len(w) > 0 and issubclass(w[-1].category, SyntaxWarning):
+                    raise ValueError("SyntaxWarning encountered")
+                return result
+            except (ValueError, SyntaxError):
+                raise
+
+    try:
+        # If var is already a float, just round and return it
+        if isinstance(var, float):
+            return round(var, decimals)
+
+        # Try to sanitize and convert the variable to a float
+        if isinstance(var, str):
+            var = var.replace("_", "")  # Remove any underscores
+
+        evaluated_var = safe_literal_eval(var)
+
+        # Convert the evaluated variable to a float and round it
+        rounded_float = round(float(evaluated_var), decimals)
+
+        return rounded_float
+    except (ValueError, SyntaxError):
+        # Return the original value if conversion is not possible
+        return var
+
+
+
 def sizeof_fmt(num, suffix='B'):
     """
     Formats data size into human readable format.
@@ -1563,6 +1606,11 @@ def anonymize_dataframe(dataframe_dbms):
             dataframe_dbms.columns = dataframe_dbms.columns.map(dbms.anonymizer)
             #dataframe_dbms.columns = anonymize_list(dataframe_dbms.columns)
 
+def natural_sort(l): 
+    convert = lambda text: int(text) if text.isdigit() else text.lower()
+    alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
+    return sorted(l, key=alphanum_key)
+
 
 import json
 from os.path import isdir, isfile, join
@@ -1603,7 +1651,7 @@ def joinDicts(d1, d2):
             filename = '{folder}/{connection}/queries.config'.format(folder=folder, connection=connection)
             copyfile(filename, folder+'/queries.config')
         except Exception as e:
-            print(e)
+            print("Exception when merging connections: {}".format(e))
     # join to single list
     # no connection name must be doubled
     connection_config = []
@@ -1615,14 +1663,16 @@ def joinDicts(d1, d2):
             if not c['name'] in connection_names:
                 connection_config.append(c)
                 connection_names.append(c['name'])
-    for connection in connection_config:
-        print("Merged connection: ", connection['name'])
+    if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+        for connection in connection_config:
+            print("Merged connection: ", connection['name'])
     # store merged config
     filename = folder+'/connections.config'
     with open(filename,'w') as inf:
         inf.write(str(connection_config))
     # merging protocols
-    print("Merge protocols")
+    if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+        print("Merge protocols")
     # load partial protocols
     protocols = []
     for connection in list_connections:
@@ -1645,7 +1695,8 @@ def joinDicts(d1, d2):
     with open(filename_protocol, 'w') as f:
         json.dump(protocol, f)
     # compare result sets
-    print("Merge result sets")
+    if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+        print("Merge result sets")
     for numQuery, query in protocol['query'].items():
         #if int(numQuery) > 3:
         #    exit()
@@ -1657,7 +1708,7 @@ def joinDicts(d1, d2):
         for connection in list_connections:
             try:
                 filename = '{folder}/{connection}/query_{numQuery}_resultset_complete_{connection}.pickle'.format(folder=folder, connection=connection, numQuery=numQuery)
-                logger.debug("Looking for", filename)
+                logger.debug("Looking for {}".format(filename))
                 if isfile(filename):
                     # result set of all runs
                     #print(connection+": ", end='')#, df)
@@ -1686,7 +1737,8 @@ def joinDicts(d1, d2):
                                 # convert datatypes
                                 #precision = query.restrict_precision
                                 precision = 2
-                                result = [[round(float(item), int(precision)) if convertToFloat(item) == float else convertToInt(item) if convertToInt(item) == item else item for item in sublist] for sublist in result]
+                                #result = [[round(float(item), int(precision)) if convertToFloat(item) == float else convertToInt(item) if convertToInt(item) == item else item for item in sublist] for sublist in result]
+                                result = [[convert_to_rounded_float(item, int(precision)) for item in sublist] for sublist in result]
                                 df = pd.DataFrame(sorted(result, key=itemgetter(*list(range(0,len(result[0]))))), columns=titles_result)
                                 #df = pd.DataFrame(result)
                                 #new_header = df.iloc[0] #grab the first row for the header
@@ -1703,7 +1755,8 @@ def joinDicts(d1, d2):
                                 # convert datatypes
                                 #precision = query.restrict_precision
                                 precision = 2
-                                storage = [[round(float(item), int(precision)) if convertToFloat(item) == float else convertToInt(item) if convertToInt(item) == item else item for item in sublist] for sublist in storage]
+                                #storage = [[round(float(item), int(precision)) if convertToFloat(item) == float else convertToInt(item) if convertToInt(item) == item else item for item in sublist] for sublist in storage]
+                                storage = [[convert_to_rounded_float(item, int(precision)) for item in sublist] for sublist in storage]
                                 df_first = pd.DataFrame(sorted(storage, key=itemgetter(*list(range(0,len(storage[0]))))), columns=titles_storage)
                                 #df_first = pd.DataFrame(data_first[numRun])
                                 #new_header = df_first.iloc[0] #grab the first row for the header
@@ -1715,14 +1768,14 @@ def joinDicts(d1, d2):
                                 #print("result", result)
                                 #print("storage", storage)
                                 if result == storage:
-                                    logger.debug("same\n")
+                                    logger.debug("same")
                                     pass
                                 #    #exit()
                                 #if numQuery=='3':
                                 #    print(df_first)
                                 #    print(df)
                                 if not df_1.empty or not df_2.empty:
-                                    logger.debug("different\n")#, df_1, df_2)
+                                    logger.debug("different")#, df_1, df_2)
                                     #print("result", result)
                                     #print("storage", storage)
                                     #exit()
@@ -1768,7 +1821,7 @@ def joinDicts(d1, d2):
                                     protocol['query'][numQuery]['resultSets'][connection] = []
                                     protocol['query'][numQuery]['warnings'][connection] = ""
             except Exception as e:
-                print(e)
+                print("Exception when merging result sets: {}".format(e))
                 #print("missing")
                 protocol['query'][numQuery]['warnings'][connection] = 'Missing'
                 traceback.print_exc()
@@ -1781,7 +1834,8 @@ def joinDicts(d1, d2):
     with open(filename_protocol, 'w') as f:
         json.dump(protocol, f)
     # merge timers
-    print("Merge timers")
+    if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+        print("Merge timers")
     # load partial timers, join and save
     timer = ['connection', 'execution', 'datatransfer']
     numQuery = 1
@@ -1808,13 +1862,14 @@ def joinDicts(d1, d2):
                 csv_file = open(filename, "w")
                 csv_file.write(csv)
                 csv_file.close()
-                logger.debug("Merged timer", filename)
+                logger.debug("Merged timer {}".format(filename))
     # merge metrics
     # copy partial metrics
     for connection in list_connections:
         folder_connection = folder+'/'+connection
         files_metrics = [f for f in listdir(folder_connection) if isfile(join(folder_connection, f)) and 'metric' in f]
         #print(folder_connection, files_metrics)
-        print("Copy Metrics", folder_connection)
+        if not benchmarker.BENCHMARKER_VERBOSE_NONE:
+            print("Copy Metrics", folder_connection)
         for file in files_metrics:
             copyfile(folder_connection+'/'+file, folder+'/'+file)

docs/Example-TPC-H.md

@@ -65,81 +65,40 @@ The script has created a result folder in the current directory containing the r
 If the `-e yes` option is used, you will see something like
 
 ```bash
-First successful query: 1
+The TPC-H Queries : This includes the reading queries of TPC-H.
+First successful query: Q1
 Limited to: ['PostgreSQL']
 Number of runs per query: 1
 Number of successful queries: 22
 Number of max. parallel clients: 1
 
 ### Errors (failed queries)
-     PostgreSQL
-Q1        False
-Q2        False
-Q3        False
-Q4        False
-Q5        False
-Q6        False
-Q7        False
-Q8        False
-Q9        False
-Q10       False
-Q11       False
-Q12       False
-Q13       False
-Q14       False
-Q15       False
-Q16       False
-Q17       False
-Q18       False
-Q19       False
-Q20       False
-Q21       False
-Q22       False
+No errors
 
 ### Warnings (result mismatch)
-     PostgreSQL
-Q1        False
-Q2        False
-Q3        False
-Q4        False
-Q5        False
-Q6        False
-Q7        False
-Q8        False
-Q9        False
-Q10       False
-Q11       False
-Q12       False
-Q13       False
-Q14       False
-Q15       False
-Q16       False
-Q17       False
-Q18       False
-Q19       False
-Q20       False
-Q21       False
-Q22       False
+No warnings
+
 ### Geometric Mean of Medians of Run Times (only successful) [s]
             average run time [s]
 DBMS
-PostgreSQL                  0.19
+PostgreSQL                  0.24
 ### Sum of Maximum Run Times per Query (only successful) [s]
             sum of max run times [s]
 DBMS
-PostgreSQL                       6.3
+PostgreSQL                      8.17
 ### Queries per Hour (only successful) [QpH] - 1*22*3600/(sum of max run times)
             queries per hour [Qph]
 DBMS
-PostgreSQL                12566.32
+PostgreSQL                 9688.82
 ### Queries per Hour (only successful) [QpH] - Sum per DBMS
             queries per hour [Qph]
 DBMS
-PostgreSQL                12566.32
+PostgreSQL                 9688.82
 ### Queries per Hour (only successful) [QpH] - (max end - min start)
             queries per hour [Qph]        formula
 DBMS
-PostgreSQL                  9900.0  1*1*22*3600/8
+PostgreSQL                  8800.0  1*1*22*3600/9
+Experiment 1720728360 has been finished
 ```
 
 ### Evaluate Results in Dashboard