From 071c0f045b823fdcda5267481847fbcd3a785ab8 Mon Sep 17 00:00:00 2001
From: HappyZ <zhuyanzi@gmail.com>
Date: Sun, 5 May 2019 22:13:01 -0500
Subject: [PATCH] refine processor, extract data from pcap

---
 libs/parser_post.py | 196 ++++++++++++++++++++++++++++++++++++++------
 libs/tshark.py      | 176 +++++++++++++++++++++++++++++++++++++++
 preprocessor.py     |  94 +++++++++++++++++++++
 visualize.m         |  18 ++++
 4 files changed, 458 insertions(+), 26 deletions(-)
 create mode 100644 libs/tshark.py
 create mode 100644 preprocessor.py
 create mode 100644 visualize.m

diff --git a/libs/parser_post.py b/libs/parser_post.py
index 6777f7d..a6ab953 100644
--- a/libs/parser_post.py
+++ b/libs/parser_post.py
@@ -4,15 +4,172 @@ from PIL import Image
 from PIL import ImageDraw
 from PIL import ImageChops
 
+from libs.tshark import Tshark
 
 
-def build_map(slam_data, map_image_data):
+RED = (255, 0, 0, 255)
+
+
+def extract_dev_from_combined(fp, minimalCounts=100, cleanup=True):
+    '''
+    extract each device data from combined file `fp`
+    '''
+    files = []
+    counters = []
+    folderpath, ext = os.path.splitext(fp)
+
+    try:
+        os.mkdir(folderpath)
+    except FileExistsError:
+        pass
+    except BaseException:
+        raise
+
+    with open(fp) as f:
+        lines = f.readlines()
+
+    for line in lines[1:]:
+        tmp = line.rstrip().split(",")
+        addr = tmp[2].replace(":", "")
+        if addr not in files:
+            files.append(addr)
+            counters.append(1)
+        else:
+            counters[files.index(addr)] += 1
+
+    for i in range(len(counters)-1, -1, -1):
+        if counters[i] < minimalCounts:
+            counters.pop(i)
+            files.pop(i)
+
+    title = lines[0].rstrip().split(",")
+    filepaths = ["{}/{}.csv".format(folderpath, file) for file in files]
+    for filepath in filepaths:
+        with open(filepath, "w") as f:
+            f.write(",".join(title[:2] + title[3:]) + "\n")
+
+    for line in lines[1:]:
+        tmp = line.rstrip().split(",")
+        addr = tmp[2].replace(":", "")
+        if addr not in files:
+            continue
+        filepath = filepaths[files.index(addr)]
+        with open(filepath, "a+") as f:
+            f.write(",".join(tmp[:2] + tmp[3:]) + "\n")
+    
+    if len(files) > 0 and cleanup:
+        os.remove(fp)
+
+    return filepaths
+
+
+def combine_sig_loc(sig_fp, loc_fp):
+    '''
+    append location to signal data
+    '''
+    filename, ext = os.path.splitext(loc_fp)
+    with open(sig_fp) as f:
+        sig_data = f.readlines()
+    with open(loc_fp) as f:
+        loc_data = f.readlines()
+    i_s = 1  # remove first line on info
+    i_l = 1  # remove first line on info
+    len_sig = len(sig_data)
+    len_loc = len(loc_data)
+    outfile = "{0}_sig.csv".format(filename.rstrip("_loc"))
+    with open(outfile, 'w') as f:
+        f.write("#x,y," + sig_data[0][1:])
+        prev_i_s = 0
+        prev_i_l = 0
+        while i_s < len_sig:
+            if prev_i_s != i_s:
+                sig_tmp = sig_data[i_s].rstrip().split(',')
+                prev_i_s = i_s
+            if prev_i_l != i_l:
+                loc_tmp = loc_data[i_l].rstrip().split(',')
+                prev_i_l = i_l
+            epoch_sig = float(sig_tmp[1])
+            epoch_loc = float(loc_tmp[2]) / 1000.0
+            x = float(loc_tmp[3])
+            y = float(loc_tmp[4])
+            if epoch_sig > epoch_loc:
+                i_l += 1
+                if i_l >= len_loc:
+                    i_l = len_loc - 1
+                continue
+            f.write("{},{},{}\n".format(x, y, ",".join(sig_tmp)))
+            i_s += 1
+    return outfile
+
+
+def translate_pcap(pcap_fp, is_csi):
+    tshark = Tshark()
+    filepath, ext = os.path.splitext(pcap_fp)
+    outputfp = "{}.csv".format(filepath)
+    if os.path.isfile(outputfp):
+        return outputfp
+    if is_csi:
+        tshark.translateCSI(pcap_fp, outputfp)
+    else:
+        tshark.translatePcap(pcap_fp, outputfp)
+    return outputfp
+
+
+def normalize_rss(rss):
+    rss = max(min(rss, -20), -85)
+    return (rss + 85) / 65.0
+
+
+def get_locs_from_parsed_sig_data(sig_data, is_csi=False):
+    # loop each loc
+    locs_data = []
+
+    if is_csi:
+        print("Not implemented yet")
+        return locs_data
+
+    for line in sig_data:
+        tmp = line.rstrip().split(",")
+        x = float(tmp[0])
+        y = float(tmp[1])
+        rss = float(tmp[4])
+        # calculate color of rss
+        color = (RED[0], RED[1], RED[2], int(255 * normalize_rss(rss)))
+        pos = (x, y, color)
+        locs_data.append(pos)
+    return locs_data
+
+
+
+def get_locs_from_slam_data(slam_data):
+    # loop each loc
+    locs_data = []
+    for line in slam_data:
+        tmp = line.rstrip().split(",")
+        robotime = float(tmp[1])
+        epoch = int(tmp[2])
+        x = float(tmp[3])
+        y = float(tmp[4])
+        yaw = float(tmp[5])
+        pos = (x, y, RED)
+        locs_data.append(pos)
+    return locs_data
+
+
+def build_map(locs_data, map_image_data):
     '''
     draws the path into the map. Returns the new map as a BytesIO
-
-    from https://github.com/dgiese/dustcloud/blob/71f7af3e2b9607548bcd845aca251326128f742c/dustcloud/build_map.py
-
+    modded from https://github.com/dgiese/dustcloud/blob/71f7af3e2b9607548bcd845aca251326128f742c/dustcloud/build_map.py
     '''
+
+    def align_xy(xy, center_x, center_y):
+        # set x & y by center of the image
+        # 20 is the factor to fit coordinates in in map
+        x = center_x + (xy[0] * 20)
+        y = center_y + (-xy[1] * 20)
+        return (x, y)
+
+
     map_image = Image.open(io.BytesIO(map_image_data))
     map_image = map_image.convert('RGBA')
 
@@ -23,8 +180,6 @@ def build_map(slam_data, map_image_data):
     # rotate image by -90°
     # map_image = map_image.rotate(-90)
 
-    red = (255, 0, 0, 255)
-    green = (0, 255, 0, 255)
     grey = (125, 125, 125, 255)  # background color
     transparent = (0, 0, 0, 0)
 
@@ -32,25 +187,12 @@ def build_map(slam_data, map_image_data):
     draw = ImageDraw.Draw(map_image)
 
     # loop each loc
-    prev_pos = None
-    for line in slam_data:
-        tmp = line.rstrip().split(",")
-        robotime = float(tmp[1])
-        epoch = int(tmp[2])
-        x = float(tmp[3])
-        y = float(tmp[4])
-        yaw = float(tmp[5])
-
-        # set x & y by center of the image
-        # 20 is the factor to fit coordinates in in map
-        x = center_x + (x * 20)
-        y = center_y + (-y * 20)
-        pos = (x, y)
-
-        if prev_pos:
-            draw.line([prev_pos, pos], red)
-
-        prev_pos = pos
+    prev_xy = None
+    for loc in locs_data:
+        xy = align_xy(loc[:2], center_x, center_y)
+        if prev_xy:
+            draw.line([prev_xy, xy], loc[2])
+        prev_xy = xy
 
     # rotate image back by 90°
     # map_image = map_image.rotate(90)
@@ -58,6 +200,7 @@ def build_map(slam_data, map_image_data):
     # crop image
     bgcolor_image = Image.new('RGBA', map_image.size, grey)
     cropbox = ImageChops.subtract(map_image, bgcolor_image).getbbox()
+    map_image = map_image.crop(cropbox)
 
     # and replace background with transparent pixels
     pixdata = map_image.load()
@@ -76,9 +219,10 @@ def test(args):
         with open(args.loc) as f:
             # skip the first line which is coumn names
             slam_data = f.readlines()[1:]
+        locs_data = get_locs_from_slam_data(slam_data)
         with open(args.map, 'rb') as f:
             map_image_data = f.read()
-        augmented_map = build_map(slam_data, map_image_data)
+        augmented_map = build_map(locs_data, map_image_data)
         filepath, ext = os.path.splitext(args.map)
         with open("{}.png".format(filepath), 'wb') as f:
             f.write(augmented_map.getvalue())
diff --git a/libs/tshark.py b/libs/tshark.py
new file mode 100644
index 0000000..ecf2f45
--- /dev/null
+++ b/libs/tshark.py
@@ -0,0 +1,176 @@
+import sys
+import time
+import numpy as np
+
+from subprocess import Popen
+from subprocess import PIPE
+
+
+class Tshark():
+    def __init__(self):
+        pass
+
+    def translateCSI(self, ifp, ofp, bw=20):
+        '''
+        extract csi from pcap file according to Nexmon hack
+        '''
+        FFTlength = 64
+        if bw is 40:
+            FFTlength = 128
+        elif bw is 80:
+            FFTlength = 256
+        cmd = [
+            '-r{0}'.format(ifp),
+            '-Tfields',
+            '-Eseparator=,',
+            '-eframe.time_epoch',
+            '-eframe.time_relative',
+            '-ewlan.ta',
+            '-eframe.len',
+            '-edata.data'
+        ]
+        p = Popen(['tshark'] + cmd, stdout=PIPE)
+        try:
+            with open(ofp, 'w') as f:
+                f.write("#txMAC,time,time_rel")
+                for i in range(1, FFTlength + 1):
+                    f.write(",sub_{0}_amp,sub_{0}_phase".format(i))
+                f.write("\n")
+                for line in p.stdout:
+                    try:
+                        t, t_rel, txMAC, frameLen, data = line.decode().rstrip().split(',')
+                    except BaseException as e:
+                        print(line)
+                        continue
+                    if not frameLen == '1076':
+                        continue
+                    t = float(t)
+                    t_rel = float(t_rel)
+                    data = data.split(':')
+                    if len(data) == 1058:
+                        ta = ':'.join(data[32:38])
+                        csi_data = data[42:]
+                    else:
+                        ta = ':'.join(data[8:14])
+                        csi_data = data[18:]
+                    csi_val = 1j * np.zeros(256)
+                    for i in range(0, len(csi_data), 4):
+                        real_p = int(
+                            "{0}{1}"
+                            .format(csi_data[i+3], csi_data[i+2]),
+                            16
+                        )
+                        if real_p > 0x7FFF:
+                            real_p -= 0x10000
+                        imag_p = int(
+                            "{0}{1}"
+                            .format(csi_data[i+1], csi_data[i+0]),
+                            16
+                        )
+                        if imag_p > 0x7FFF:
+                            imag_p -= 0x10000
+                        csi_val[i / 4] = np.complex(real_p, imag_p)
+                    if FFTlength < 256:
+                        cmplx_bw = np.fft.fftshift(csi_val[1:(FFTlength+1)])
+                    else:
+                        cmplx_bw = np.fft.fftshift(csi_val[:-1])
+                    # set 0 to null carriers
+                    if bw is 20:
+                        cmplx_bw[:4] = 0
+                        cmplx_bw[32] = 0
+                        cmplx_bw[61:64] = 0
+                    elif bw is 40:
+                        cmplx_bw[:6] = 0
+                        cmplx_bw[63:66] = 0
+                        cmplx_bw[123:128] = 0
+                    elif bw is 80:
+                        cmplx_bw[:6] = 0
+                        cmplx_bw[127:130] = 0
+                        cmplx_bw[251:256] = 0
+                    f.write("{0},{1:.4f},{2:.4f}".format(ta, t, t_rel))
+                    for each in zip(np.abs(cmplx_bw), np.angle(cmplx_bw)):
+                        f.write(",{0:.6f},{1:.4f}".format(each[0], each[1]))
+                    f.write("\n")
+        except KeyboardInterrupt:
+            p.kill()
+        except Exception:
+            raise
+
+    def translatePcap(self, ifp, ofp):
+        '''
+        translate pcap data into desired format via tshark
+        '''
+        cmd = [
+            '-r{0}'.format(ifp),
+            '-Tfields',
+            '-Eseparator=,',
+            '-ewlan.ta',
+            '-eframe.time_epoch',
+            '-eframe.time_relative',
+            '-ewlan_radio.signal_dbm',
+            '-ewlan_radio.noise_dbm',
+            '-eframe.len',
+            '-eradiotap.channel.freq',
+            '-ewlan.fc.type_subtype',
+            '-ewlan.frag'
+        ]
+        p = Popen(['tshark'] + cmd, stdout=PIPE)
+        try:
+            with open(ofp, 'w') as f:
+                f.write("#txMAC,time,time_rel,RSS,noise,frameLen,channelFreq,type,fragNum\n")
+                for line in p.stdout:
+                    tmp = line.decode()
+                    if tmp.split(",")[0]:
+                        f.write(tmp)
+        except KeyboardInterrupt:
+            p.kill()
+        except Exception:
+            raise
+
+
+def test(args):
+    if args.outf is None:
+        print("Must specify output filepath")
+        return
+
+    tshark = Tshark()
+
+    if args.rss:
+        tshark.translatePcap(args.rss, args.outf)
+
+    elif args.csi:
+        tshark.translateCSI(args.csi, args.outf)
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description='tshark test'
+    )
+
+    parser.add_argument(
+        '-rss', '--rss',
+        dest='rss',
+        default=None,
+        help='Specify RSS pcap file path'
+    )
+
+    parser.add_argument(
+        '-csi', '--csi',
+        dest='csi',
+        default=None,
+        help='Specify CSI pcap file path'
+    )
+
+    parser.add_argument(
+        '-o', '--outf',
+        dest='outf',
+        default=None,
+        help='Specify output filepath'
+    )
+
+    args, __ = parser.parse_known_args()
+
+    test(args)
+
diff --git a/preprocessor.py b/preprocessor.py
new file mode 100644
index 0000000..4f20f51
--- /dev/null
+++ b/preprocessor.py
@@ -0,0 +1,94 @@
+import os
+import argparse
+
+from libs.parser_post import build_map
+from libs.parser_post import translate_pcap
+from libs.parser_post import combine_sig_loc
+from libs.parser_post import get_locs_from_slam_data
+from libs.parser_post import get_locs_from_parsed_sig_data
+from libs.parser_post import extract_dev_from_combined
+
+
+def get_files(folder):
+    files = os.listdir(folder)
+    f_map_image = None
+    f_loc_est = None
+    f_sig_data = None
+    is_csi = False
+    for file in files:
+        if '.pcap' in file:
+            f_sig_data = "{0}/{1}".format(folder, file)
+            if "csi" in file:
+                is_csi = True
+        elif 'loc.csv' in file:
+            f_loc_est = "{0}/{1}".format(folder, file)
+        elif 'map.ppm' in file:
+            f_map_image = "{0}/{1}".format(folder, file)
+    if f_loc_est is None or f_map_image is None:
+        f_loc_est = None
+        f_map_image = None
+        f_sig_data = None
+    return f_map_image, f_loc_est, f_sig_data, is_csi
+
+
+
+def generate_map(f_map, f_loc, f_sig_extracted, is_csi):
+    '''
+    generate maps for path and signals
+    '''
+
+    # get map image
+    with open(f_map, 'rb') as f:
+        map_image_data = f.read()
+
+    # build a general map without signal
+    with open(f_loc) as f:
+        # skip the first line which is coumn names
+        slam_data = f.readlines()[1:]
+    augmented_map = build_map(
+        get_locs_from_slam_data(slam_data),
+        map_image_data
+    )
+    filepath, ext = os.path.splitext(f_map)
+    with open("{}.png".format(filepath), 'wb') as f:
+        f.write(augmented_map.getvalue())
+
+    for f_each in f_sig_extracted:
+        with open(f_each) as f:
+            # skip the first line which is coumn names
+            parsed_sig_data = f.readlines()[1:]
+        augmented_map = build_map(
+            get_locs_from_parsed_sig_data(parsed_sig_data, is_csi),
+            map_image_data
+        )
+        filepath, ext = os.path.splitext(f_each)
+        with open("{}.png".format(filepath), 'wb') as f:
+            f.write(augmented_map.getvalue())
+
+
+def main(args):
+    if not os.path.isdir(args.folder):
+        print("Err: folder {} does not exist".format(args.folder))
+        exit(2)
+    f_map, f_loc, f_sig, is_csi = get_files(args.folder)
+    if f_map is None or f_loc is None or f_sig is None:
+        print("Err: desired files not exist")
+        exit(2)
+    # parse pcap into csv, and add location if it has one
+    f_sig_parsed = translate_pcap(f_sig, is_csi)
+    f_sig_combined = combine_sig_loc(f_sig_parsed, f_loc)
+    f_sig_extracted = extract_dev_from_combined(f_sig_combined, minimalCounts=100)
+    # generate path in map for visualization
+    generate_map(f_map, f_loc, f_sig_extracted, is_csi)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Data Pre-Processor'
+    )
+    parser.add_argument(
+        dest='folder',
+        help='Specify folder path of data'
+    )
+    args, __ = parser.parse_known_args()
+    main(args)
\ No newline at end of file
diff --git a/visualize.m b/visualize.m
new file mode 100644
index 0000000..350dbb1
--- /dev/null
+++ b/visualize.m
@@ -0,0 +1,18 @@
+rawdata = readtable('./test/20190505_170223_sig/98fc11691fc5.csv');
+data = table2array(rawdata(:, [1,2,4,5,9]));
+unique_types = unique(data(:,5));
+
+for j = 1:size(unique_types, 1)
+    figure(j); clf;
+    logistics = data(:,5) == unique_types(j);
+    tmpdata = data(logistics, :);
+    unique_xys = unique(tmpdata(:, 1:2), 'row');
+    tmpdata_avg = [];
+    for i = size(unique_xys, 1):-1:1
+        xy_logistics = unique_xys(i, 1:2) == tmpdata(:, 1:2);
+        xy_logistics = xy_logistics(:,1) & xy_logistics(:,2);
+        tmpdata_avg(i, :) = [unique_xys(i, 1:2), mean(tmpdata(xy_logistics, 3:end), 1)];
+    end
+    scatter3(tmpdata_avg(:,1), tmpdata_avg(:,2), tmpdata_avg(:,3), 100, tmpdata_avg(:,4), '.');
+    colorbar;
+end
\ No newline at end of file