diff --git a/opencellid_parsing/001.data_extraction.py b/opencellid_parsing/001.data_extraction.py new file mode 100644 index 0000000..3021703 --- /dev/null +++ b/opencellid_parsing/001.data_extraction.py @@ -0,0 +1,184 @@ +#!/usr/bin/python + +import os +import sys +import pickle + +# global +folderpath = sys.argv[1] # folder containing all measurement_xx.csv +celltowers_fp = sys.argv[2] # csv file containing all cell_tower info + + +############################ +# get cell towers info +############################ +celltowers_cared = {} +celltowers_not_cared = {} + +try: + firstLine = True + with open(celltowers_fp, 'r') as f: + lines = f.readlines() + for line in lines: + if firstLine: + firstLine = False + continue + tmp = line.rstrip().split(',') + radio = tmp[0] # Network type. One of the strings GSM, UMTS, LTE or CDMA. + mcc = tmp[1] # mobile country code + net = tmp[2] # For GSM, UMTS and LTE networks, this is the Mobile Network Code (MNC). + # For CDMA networks, this is the System IDentification number (SID). + area = tmp[3] # Location Area Code (LAC) for GSM and UMTS networks. + # Tracking Area Code (TAC) for LTE networks. + # Network IDenfitication number (NID) for CDMA networks. + cell = tmp[4] # Cell ID (CID) for GSM and LTE networks. + # UTRAN Cell ID / LCID for UMTS networks, + # which is the concatenation of 2 or 4 bytes of + # Radio Network Controller (RNC) code and 4 bytes of Cell ID. + # Base station IDentifier number (BID) for CDMA networks. + unit = tmp[5] # Primary Scrambling Code (PSC) for UMTS networks. + # Physical Cell ID (PCI) for LTE networks. + # An empty value for GSM and CDMA networks. + lon = float(tmp[6]) if not tmp[6] == '' else float('nan') + lat = float(tmp[7]) if not tmp[7] == '' else float('nan') + cellrange = tmp[8] # Estimate of cell range, in meters. + samples = tmp[9] # Total number of measurements assigned to the cell tower + changeable = int(tmp[10]) + created = tmp[11] + updated = tmp[12] + averageSignal = tmp[13] + key = "{}_{}_{}_{}_{}".format(radio, mcc, net, area, cell) + mydict = celltowers_not_cared + if changeable is 0: + mydict = celltowers_cared + mydict[key] = { + 'location': (lon, lat), + 'mcc': int(mcc) if not mcc == '' else None, + 'net': int(net) if not net == '' else None, + 'area': int(area) if not area == '' else None, + 'cell': int(cell) if not cell == '' else None, + 'unit': int(unit) if not unit == '' else None, + 'cellrange': cellrange, + 'samples': samples, + 'counter': 0 + } +except BaseException: + raise + + +############################ +# extract measurements +############################ +headline = '#lon,lat,sig,measure_at,upload_at,rating,speed,direction' +headlines = { + 'UMTS': headline + ',rnc,cidshort,psc', + 'GSM': headline + ',timingadv', + 'LTE': headline + ',timingadv,tac,pci', + 'CDMA': headline + ',sid,nid,bid', +} + +outputfolder = folderpath.rstrip('/') + '_extracted' +if not os.path.isdir(outputfolder): + os.makedirs(outputfolder) +else: + print("folder exists, will not overwrite but append") + +files = [file for file in os.listdir(folderpath) if '.csv' in file] +for file in sorted(files, key=lambda x: x.rstrip('.csv').split('_')[-1]): + print("looking at file: {}".format(file)) + with open("{}/{}".format(folderpath, file), 'r') as f: + lines = f.readlines() + for line in lines: + tmp = line.rstrip().split(',') + # mobile country code + mcc = int(tmp[0]) if not tmp[0] == '' else float('nan') + # For GSM, UMTS and LTE networks, this is the Mobile Network Code (MNC). + # For CDMA networks, this is the System IDentification number (SID). + net = int(tmp[1]) if not tmp[1] == '' else float('nan') + # Location Area Code (LAC) for GSM and UMTS networks. + # Tracking Area Code (TAC) for LTE networks. + # Network IDenfitication number (NID) for CDMA networks. + area = int(tmp[2]) if not tmp[2] == '' else float('nan') + # Cell ID (CID) for GSM and LTE networks. + # UTRAN Cell ID / LCID for UMTS networks, + # which is the concatenation of 2 or 4 bytes of + # Radio Network Controller (RNC) code and 4 bytes of Cell ID. + # Base station IDentifier number (BID) for CDMA networks. + cell = int(tmp[3]) if not tmp[3] == '' else float('nan') + # measurement location + lon = float(tmp[4]) if not tmp[4] == '' else float('nan') + lat = float(tmp[5]) if not tmp[5] == '' else float('nan') + # signal level + sig = int(tmp[6]) if not tmp[6] == '' else float('nan') + # measured and uploaded-to-opencellid time + measure_t = int(tmp[7]) if not tmp[7] == '' else float('nan') + created_t = int(tmp[8]) if not tmp[8] == '' else float('nan') + # GPS quality/accuracy information (metres) + rating = float(tmp[9]) if not tmp[9] == '' else float('nan') + # Speed when creating the measurement; both metres/second and km/h is accepted. + speed = float(tmp[10]) if not tmp[10] == '' else float('nan') + # Heading direction of the phone / telematics device at the moment + # the measurement was created (0=north, 90=east, 180=south, 270=west) + direction = float(tmp[11]) if not tmp[11] == '' else float('nan') + # Network type. One of the strings GSM, UMTS, LTE or CDMA + radio = tmp[12] + # Timing advance; only for GSM and LTE + timingadv = int(tmp[13]) if not tmp[13] == '' else float('nan') + # Radio network controller; only for UMTS + rnc = int(tmp[14]) if not tmp[14] == '' else float('nan') + # Cell id (short); only for UMTS + cidshort = int(tmp[15]) if not tmp[15] == '' else float('nan') + # Primary scrambling code; only for UMTS + psc = int(tmp[16]) if not tmp[16] == '' else float('nan') + # Tracking area code; only for LTE + tac = int(tmp[17]) if not tmp[17] == '' else float('nan') + # Physical cell id; only for LTE + pci = int(tmp[18]) if not tmp[18] == '' else float('nan') + # System identifier; only for CDMA + sid = int(tmp[19]) if not tmp[19] == '' else float('nan') + # Network id; only for CDMA + nid = int(tmp[20]) if not tmp[20] == '' else float('nan') + # Base station id; only for CDMA + bid = int(tmp[21]) if not tmp[21] == '' else float('nan') + # format dataline + dataline = ( + "{:.6f},{:.6f},{},{},{},{:.2f},{:.2f},{:.0f}" + .format(lon, lat, sig, measure_t, created_t, rating, speed, direction) + ) + if radio == 'UMTS': + dataline += ",{},{},{}".format(rnc, cidshort, psc) + elif radio == 'GSM': + dataline += ",{}".format(timingadv) + elif radio == 'LTE': + dataline += ",{},{},{}".format(timingadv, tac, pci) + elif radio == 'CDMA': + dataline += ",{},{},{}".format(sid, nid, bid) + # get key in previous dict + key = "{}_{}_{}_{}_{}".format(radio, mcc, net, area, cell) + # folder level: outputfolder/with(out)_groundtruth/radio/mcc/net/area/ + if key in celltowers_cared: + mydict = celltowers_cared + outputsubfolder = "{}/with_groundtruth/{}/{}/{}/{}".format(outputfolder, radio, mcc, net, area) + elif key in celltowers_not_cared: + mydict = celltowers_not_cared + outputsubfolder = "{}/without_groundtruth/{}/{}/{}/{}".format(outputfolder, radio, mcc, net, area) + else: + mydict = {} + outputsubfolder = "{}/unseen/{}/{}/{}/{}".format(outputfolder, radio, mcc, net, area) + try: + if not os.path.isdir(outputsubfolder): + os.makedirs(outputsubfolder) + except BaseException: + raise + gt_loc = mydict.get(key, {}).get('location', (float('nan'), float('nan'))) + filepath = "{}/{}_{:.6f}_{:.6f}.csv".format(outputsubfolder, cell, gt_loc[0], gt_loc[1]) + if not os.path.isfile(filepath): + with open(filepath, 'w') as of: + of.write(headlines.get(radio, headline) + '\n') + with open(filepath, 'a') as of: + of.write(dataline + '\n') + if key in mydict: + mydict[key]['counter'] += 1 + +pickle.dump(celltowers_cared, open(celltowers_fp.replace('.csv', '_with_gt.pickle'), 'wb')) +pickle.dump(celltowers_not_cared, open(celltowers_fp.replace('.csv', '_no_gt.pickle'), 'wb'))