more log

experiment/orchestrator.py

@@ -8,6 +8,7 @@ import subprocess
 import threading
 import argparse
 import queue
+import datetime
 
 EMULATORS = [f"root34-{i}" for i in range(16)]
 ANDROID_IMG = "system-images;android-34;default;x86_64"
@@ -239,8 +240,11 @@ def worker(emu: str, apklist: queue.Queue[str], out_folder: Path, script: Path):
     script_env = os.environ.copy()
     script_env["ANDROID_HOME"] = str(ANDROID_HOME)
     proc_emu = restore_emu(emu, None)
+    marked_done = False
+    try:
         while True:
             apk = apklist.get()
+            marked_done = False
             folder_name = apk.split("/")[-1].removesuffix(".apk")
             folder = out_folder / folder_name
 
@@ -261,6 +265,7 @@ def worker(emu: str, apklist: queue.Queue[str], out_folder: Path, script: Path):
                 else:
                     # We already have a valid result, mark task done and skip xp
                     apklist.task_done()
+                    marked_done = True
                     continue
 
             folder.mkdir(parents=True)
@@ -302,7 +307,9 @@ def worker(emu: str, apklist: queue.Queue[str], out_folder: Path, script: Path):
                     i += 1
 
                 print(f"emulator-{console_port} running")
-            fp_anly_stdout.write(f"START ANALYSIS: {apk=}, emulator-{console_port}\n")
+                fp_anly_stdout.write(
+                    f"START ANALYSIS: {apk=}, emulator-{console_port}\n"
+                )
                 # should help debuging:
                 subprocess.run(
                     [ADB, "devices"],
@@ -313,7 +320,13 @@ def worker(emu: str, apklist: queue.Queue[str], out_folder: Path, script: Path):
                 # Run script
                 try:
                     subprocess.run(
-                    ["bash", str(script), apk, f"emulator-{console_port}", str(folder)],
+                        [
+                            "bash",
+                            str(script),
+                            apk,
+                            f"emulator-{console_port}",
+                            str(folder),
+                        ],
                         env=script_env,
                         stdout=fp_anly_stdout,
                         stderr=fp_anly_stderr,
@@ -329,6 +342,14 @@ def worker(emu: str, apklist: queue.Queue[str], out_folder: Path, script: Path):
                 with (folder / "emu").open("w") as fp:
                     fp.write(f"Used emulator {emu}:  emulator-{console_port}")
             apklist.task_done()
+            marked_done = True
+    except Exception as e:
+        msg = f"[{datetime.datetime.now()}] worker for {emu} (emulator-{console_port}) terminated after {e}"
+        print(msg)
+        with (out_folder / "worker_{emu}").open("w") as fp:
+            fp.write(msg)
+        if not marked_done:
+            apklist.task_done()
 
 
 def run(apklist: list[str], out_folder: Path, script: Path):