speed up the processing

libs/parser_post.py

@@ -14,8 +14,7 @@ def extract_dev_from_combined(fp, minimalCounts=100, cleanup=True):
     '''
     extract each device data from combined file `fp`
     '''
-    files = []
-    counters = []
+    files = {}
     folderpath, ext = os.path.splitext(fp)
 
     try:
@@ -32,30 +31,24 @@ def extract_dev_from_combined(fp, minimalCounts=100, cleanup=True):
         tmp = line.rstrip().split(",")
         addr = tmp[2].replace(":", "")
         if addr not in files:
-            files.append(addr)
-            counters.append(1)
-        else:
-            counters[files.index(addr)] += 1
+            files[addr] = []
+        files[addr].append(",".join(tmp[:2] + tmp[3:]))
 
-    for i in range(len(counters)-1, -1, -1):
-        if counters[i] < minimalCounts:
-            counters.pop(i)
-            files.pop(i)
+    for addr in list(files.keys()):
+        if len(files[addr]) < minimalCounts:
+            del files[addr]
 
     title = lines[0].rstrip().split(",")
-    filepaths = ["{}/{}.csv".format(folderpath, file) for file in files]
-    for filepath in filepaths:
-        with open(filepath, "w") as f:
-            f.write(",".join(title[:2] + title[3:]) + "\n")
+    headline = ",".join(title[:2] + title[3:]) + "\n"
+    filepaths = []
 
-    for line in lines[1:]:
-        tmp = line.rstrip().split(",")
-        addr = tmp[2].replace(":", "")
-        if addr not in files:
-            continue
-        filepath = filepaths[files.index(addr)]
-        with open(filepath, "a+") as f:
-            f.write(",".join(tmp[:2] + tmp[3:]) + "\n")
+    for addr in files.keys():
+        filepath = "{}/{}.csv".format(folderpath, addr)
+        filepaths.append(filepath)
+        with open(filepath, "w") as f:
+            f.write(headline)
+            for line in files[addr]:
+                f.write("{}\n".format(line))
 
     if len(files) > 0 and cleanup:
         os.remove(fp)

pull_data_from_phone.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+PHONEPATH=$1
+LOCALFOLDER=$2
+
+mkdir -p $LOCALFOLDER
+
+for file in $(adb shell "ls $PHONEPATH" | tr -d '\r'); do
+    if [[ $PHONEPATH = *"*" ]]; then
+        adb pull "${PHONEPATH%%/*}/$file" $LOCALFOLDER/
+        adb shell rm "${PHONEPATH%%/*}/$file"
+    else
+        adb pull "$PHONEPATH/$file" $LOCALFOLDER/
+        adb shell rm "$PHONEPATH/$file"
+    fi
+done

visualize.m

@@ -1,4 +1,4 @@
-rawdata = readtable('./example/20190505_170223_sig/98fc11691fc5.csv');
+rawdata = readtable('./example/20190505_170223_sig/98ded0624482.csv');
 data = table2array(rawdata(:, [1,2,4,5,9]));
 unique_types = unique(data(:,5));