debug cuda

examples/pytorch_bert.py

@@ -32,9 +32,6 @@ def run_benchmark():
 
 
 if __name__ == "__main__":
-    level = os.environ.get("LOG_LEVEL", "INFO")
-    print("level", level)
-    raise Exception("This is a test exception")
     to_file = os.environ.get("LOG_TO_FILE", "0") == "1"
     setup_logging(level=level, to_file=to_file, prefix="MAIN-PROCESS")
 

examples/pytorch_gpt2.py

@@ -65,6 +65,8 @@ def run_benchmark(weight_config: str):
 
 if __name__ == "__main__":
     level = os.environ.get("LOG_LEVEL", "INFO")
+    print("level", level)
+    raise Exception("This is a test exception")
     to_file = os.environ.get("LOG_TO_FILE", "0") == "1"
     setup_logging(level=level, to_file=to_file, prefix="MAIN-PROCESS")
 

tests/test_examples.py

@@ -152,7 +152,27 @@ def test_yaml_config(config_name):
 def execute_python_script(script_name):
     script_path = EXAMPLES_DIR / script_name
     # Run the example file as a separate process
-    result = subprocess.run([sys.executable, str(script_path)], capture_output=True, text=True)
+    process = subprocess.Popen([sys.executable, str(script_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    # Capture and display output in real-time
+    while True:
+        output = process.stdout.readline()
+        if output == '' and process.poll() is not None:
+            break
+        if output:
+            print(output.strip())
+            sys.stdout.flush()
+
+    # Capture any remaining output
+    stdout, stderr = process.communicate()
+
+    # Create a result object similar to subprocess.run
+    result = subprocess.CompletedProcess(
+        args=[sys.executable, str(script_path)],
+        returncode=process.returncode,
+        stdout=stdout,
+        stderr=stderr
+    )
 
     # Check that the process completed successfully (return code 0)
     assert result.returncode == 0, f"Script {script_path} failed with error:\n{result.stderr}"