From e749c877f104d65560a081f95d6a3c27727a864f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lukas=20W=C3=B6lfer?= <lukas.woelfer@rwth-aachen.de>
Date: Thu, 13 Feb 2020 20:00:34 +0100
Subject: [PATCH] Changed benchmark output

---
 .vscode/tasks.json    |  4 ++-
 benchmark/timing.py   |  5 ++--
 benchmark/unsolved.py | 10 +++++--
 benchmark/util.py     | 67 ++++++++++++++++++++++++++++++-------------
 4 files changed, 60 insertions(+), 26 deletions(-)

diff --git a/.vscode/tasks.json b/.vscode/tasks.json
index e62f207..5371028 100644
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -10,7 +10,8 @@
             "args": [
                 "-m",
                 "benchmark.timing"
-            ]
+            ],
+            "problemMatcher": []
         },
         {
             "label": "Unsolved benchmark",
@@ -20,6 +21,7 @@
                 "-m",
                 "benchmark.unsolved"
             ],
+            "problemMatcher": []
         }
     ]
 }
\ No newline at end of file
diff --git a/benchmark/timing.py b/benchmark/timing.py
index 9abe032..650bfb0 100644
--- a/benchmark/timing.py
+++ b/benchmark/timing.py
@@ -27,8 +27,9 @@ def main() -> None:
         result = pool.imap_unordered(
             run_benchmark, [Path(benchmark) for benchmark in benchmark_files]
         )
-        for current_result in result:
-            print(current_result)
+        pool.close()
+        pool.join()
+        
 
 
 if __name__ == "__main__":
diff --git a/benchmark/unsolved.py b/benchmark/unsolved.py
index 06b3d9c..8b1a0b6 100644
--- a/benchmark/unsolved.py
+++ b/benchmark/unsolved.py
@@ -28,13 +28,17 @@ benchmark_files = [
 ]
 
 
+def runner(benchmark: Path) -> None:
+    run_benchmark(benchmark, timeout=60)
+
+
 def main() -> None:
     with multiprocessing.Pool() as pool:
         result = pool.imap_unordered(
-            run_benchmark, [Path(benchmark) for benchmark in benchmark_files]
+            runner, [Path(benchmark) for benchmark in benchmark_files]
         )
-        for current_result in result:
-            print(current_result)
+        pool.close()
+        pool.join()
 
 
 if __name__ == "__main__":
diff --git a/benchmark/util.py b/benchmark/util.py
index db27517..503199d 100644
--- a/benchmark/util.py
+++ b/benchmark/util.py
@@ -6,31 +6,58 @@ import cv2
 import shenzhen_solitaire.card_detection.configuration as configuration
 import shenzhen_solitaire.solver.solver as solver
 from shenzhen_solitaire.card_detection.board_parser import parse_board
+from typing import Callable, List, Tuple
 
 
-def run_benchmark(benchmark: Path) -> str:
+class SingleTimer:
+    def __init__(self, name: str, callback: Callable[[str, float], None]):
+        self.name = name
+        self.callback = callback
+        self.start = 0.0
+
+    def __enter__(self) -> None:
+        self.start = time.time()
+        return
+
+    def __exit__(self, *args) -> None:
+        self.callback(self.name, time.time() - self.start)
+        return
+
+
+class BenchmarkTimer:
+    def __init__(self) -> None:
+        self.timing: List[Tuple[str, float]] = []
+
+    def addTiming(self, name: str, duration: float) -> None:
+        self.timing.append((name, duration))
+
+    def stopwatch(self, name: str) -> SingleTimer:
+        return SingleTimer(name, self.addTiming)
+
+    @property
+    def timings(self) -> List[float]:
+        return [x[1] for x in self.timing]
+
+
+def run_benchmark(benchmark: Path, *, timeout: float = 10) -> None:
     result = ""
     result += f"{benchmark}:\n"
-    read_file_time = time.time()
-    image = cv2.imread(str(benchmark))
-    load_config_time = time.time()
-    result += f"\tLoad image:  {load_config_time - read_file_time:5.2f}\n"
+    my_timer = BenchmarkTimer()
+    with my_timer.stopwatch("Load image"):
+        image = cv2.imread(str(benchmark))
 
-    conf = configuration.load("test_config.zip")
-    parse_board_time = time.time()
-    result += f"\tLoad config: {parse_board_time - load_config_time:5.2f}\n"
+    with my_timer.stopwatch("Load configuration"):
+        conf = configuration.load("test_config.zip")
 
-    board = parse_board(image, conf)
-    solve_time = time.time()
-    result += f"\tParse image: {solve_time - parse_board_time:5.2f}\n"
+    with my_timer.stopwatch("Parse board"):
+        board = parse_board(image, conf)
 
-    solution_iterator = next(solver.solve(board, timeout=10), None)
-    finished_time = time.time()
-    result += f"\tSolve board: {finished_time - solve_time:5.2f}\n"
+    with my_timer.stopwatch("Solve board"):
+        solution_iterator = next(solver.solve(board, timeout=timeout), None)
 
-    assert board.check_correct()
-    if solution_iterator is None:
-        result += "\tSolution timed out\n"
-    else:
-        result += f"\tSolved in {len(list(solution_iterator))} steps\n"
-    return result
+    solved_string = (
+        "[" + ("Solved" if solution_iterator is not None else "Unsolved") + "]"
+    )
+    timings_string = "\t".join(f"{x:>5.2f}" for x in my_timer.timings)
+    print(f"{solved_string:<10} {benchmark}")
+    print(f"{timings_string}")