first commit

rasta_exp/utils/couch_to_dir.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+
+import couchdb
+import json
+import sys
+import os
+import configparser
+_config = configparser.ConfigParser()
+_config.read("settings.ini")
+_cfg = _config["Couch"]
+couch_srv = couchdb.Server(f"http://{_cfg.get('user')}:{_cfg.get('password')}@{_cfg.get('host')}:{_cfg.getint('port')}/")
+
+
+db_name = sys.argv[1]
+dir_name = sys.argv[2]
+
+couch_db = couch_srv['rasta-' + db_name]
+
+if os.path.isdir(dir_name):
+    raise Exception(f"Path {dir_name} already exists. Aborting")
+
+os.makedirs(dir_name)
+
+for uid in couch_db:
+    (sha, task) = uid.split('_-_')
+    report = couch_db[uid]
+    with open(os.path.join(dir_name, uid), 'w') as f_out:
+        json.dump(report, f_out)
+

rasta_exp/utils/gen_tasks.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set_name=$1
+
+parallel echo {1}_-_{2} :::: dataset/${set_name} tools  > ${set_name}_all

rasta_exp/utils/get_extraneous.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+#
+# Just to check that no report ended up in the wrong DB
+
+import couchdb
+import json
+import sys
+import os
+import configparser
+_config = configparser.ConfigParser()
+_config.read("settings.ini")
+_cfg = _config["Couch"]
+couch_srv = couchdb.Server(f"http://{_cfg.get('user')}:{_cfg.get('password')}@{_cfg.get('host')}:{_cfg.getint('port')}/")
+
+
+set_name = sys.argv[1]
+
+couch_db = couch_srv['rasta-' + set_name]
+
+todo = set()
+with open(f"{set_name}_all", 'r') as f_in:
+    for line in f_in.readlines():
+        todo.add(line.strip())
+
+
+for uid in couch_db:
+    #(sha, task) = uid.split('_-_')
+    if uid in todo:
+        continue
+    else:
+        print(uid)
+

rasta_exp/utils/get_finished.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+import couchdb
+import json
+import sys
+import os
+import configparser
+_config = configparser.ConfigParser()
+_config.read("settings.ini")
+_cfg = _config["Couch"]
+couch_srv = couchdb.Server(f"http://{_cfg.get('user')}:{_cfg.get('password')}@{_cfg.get('host')}:{_cfg.getint('port')}/")
+
+
+db_name = sys.argv[1]
+
+couch_db = couch_srv[db_name]
+
+
+for uid in couch_db:
+    (sha, task) = uid.split('_-_')
+    report = couch_db[uid]
+    if report and 'tool-status' in report:
+        if report['tool-status'] == 'FINISHED':
+            print(uid)
+

rasta_exp/utils/get_missing.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+
+import couchdb
+import json
+import sys
+import os
+import configparser
+_config = configparser.ConfigParser()
+_config.read("settings.ini")
+_cfg = _config["Couch"]
+couch_srv = couchdb.Server(f"http://{_cfg.get('user')}:{_cfg.get('password')}@{_cfg.get('host')}:{_cfg.getint('port')}/")
+
+
+set_name = sys.argv[1]
+
+couch_db = couch_srv['rasta-' + set_name]
+
+with open(f"{set_name}_all", 'r') as f_in:
+    for line in f_in.readlines():
+        line = line.strip()
+        if line in couch_db:
+            continue
+        else:
+            print(line)
+

rasta_exp/utils/get_status_stats.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+
+import couchdb
+import json
+import sys
+import os
+import configparser
+_config = configparser.ConfigParser()
+_config.read("settings.ini")
+_cfg = _config["Couch"]
+couch_srv = couchdb.Server(f"http://{_cfg.get('user')}:{_cfg.get('password')}@{_cfg.get('host')}:{_cfg.getint('port')}/")
+
+
+set_name = sys.argv[1]
+
+couch_db = couch_srv['rasta-' + set_name]
+
+tools = set()
+with open('tools', 'r') as f:
+    for line in f.readlines():
+        tools.add(line.strip())
+
+
+set_size = 0
+with open(f'dataset/{set_name}', 'r') as f:
+    set_size = len(f.readlines())
+
+task_num = set_size * len(tools)
+results = dict()
+for tool in tools:
+    results[tool] = {'FINISHED': 0, 'TIMEOUT': 0, 'FAILED': 0, 'UNKNOWN': 0, 'MISSING': 0} 
+
+for uid in couch_db:
+    (sha, tool) = uid.split('_-_')
+
+    report = couch_db[uid]
+    results[tool][report['tool-status']] += 1
+#    if report and 'tool-status' in report:
+#        if report['tool-status'] == 'FINISHED':
+#            print(uid)
+
+global_res = {'FINISHED': 0, 'TIMEOUT': 0, 'FAILED': 0, 'UNKNOWN': 0, 'MISSING': 0} 
+for tool in sorted(list(tools)):
+    results[tool]['MISSING'] = set_size - results[tool]['FINISHED'] -  results[tool]['FAILED'] -  results[tool]['TIMEOUT'] -  results[tool]['UNKNOWN']
+    for status in ['FINISHED', 'TIMEOUT', 'FAILED', 'UNKNOWN', 'MISSING']:
+        results[tool][f"{status}_rate"] = (results[tool][status] * 100) / set_size
+        global_res[status] += results[tool][status]
+
+for status in ['FINISHED', 'TIMEOUT', 'FAILED', 'UNKNOWN', 'MISSING']:
+    global_res[f"{status}_rate"] = (global_res[status] * 100) / task_num
+
+
+print(';'.join( [set_name] + sorted(list(tools)) + ['TOTAL']) )
+for status in ['FINISHED', 'FAILED', 'TIMEOUT', 'UNKNOWN', 'MISSING', 'FINISHED_rate', 'FAILED_rate', 'TIMEOUT_rate', 'UNKNOWN_rate', 'MISSING_rate']:
+    _tmp_row = []
+    _tmp_row.append(status)
+    for tool in sorted(list(tools)):
+        _tmp = results[tool][status]
+        if status.endswith('rate'):
+            _tmp_row.append(f"{_tmp:.2f}")
+        else:
+            _tmp_row.append(f"{_tmp}")
+    if status.endswith('rate'):
+        _tmp_row.append(f"{global_res[status]:.2f}")
+    else:
+        _tmp_row.append(f"{global_res[status]}")
+    print(";".join(_tmp_row))
+
+#print(results)
+#print(global_res)

rasta_exp/utils/init.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set_name=$1
+
+
+for set_num in `seq 0 9`; do
+	set_name=set${set_num}
+	utils/gen_tasks.sh ${set_name}
+done
+utils/gen_tasks.sh drebin

rasta_exp/utils/make_all_stats.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+utils/get_status_stats.py drebin > drebin_stats
+seq 0 9 | parallel -j10 'utils/get_status_stats.py set{} > set{}_stats'