Fix split-and-strip approach to parts in partitions

When a part returned by Louvain (edges are based on distances, not
time; thus 1 part can contain two overtakes, if they happen to be
roughly at the same lat. dist.), split it to subparts around time
gaps, then discard subparts which are too small and cannot be
overtakes. Finally, take the the leftmost (closes to press) subpart as
the event.

Author: Jakub Sliacan

__main__.py

@@ -27,7 +27,7 @@
     # presses
     press_starts, press_lengths = box.get_press_lengths_and_starts(ldata)
     date_string = csv_file.split("/")[-1][:8]
-
+    
     b_partitions, a_partitions, b_modularities, a_modularities = mod.get_partitions(ldata, press_starts, press_lengths)
 
     for j, b_parts in enumerate(b_partitions):
@@ -52,24 +52,21 @@
             lp.sort() # should already be sorted
 
             # skip the maxed-out readings
-            if min([lat_dists[x] for x in p]) > 500:
+            if min([lat_dists[x] for x in p]) > 450:
                 continue
 
             # skip the readings stuck too low
             if max([lat_dists[x] for x in p]) < 50: 
                 continue
 
-            # only take 1st part of a partition if there's a big gap in time
-            # TODO: append the latter part back to partitions, and adjust part_and_size_pairs.
-            for i in range(1,s):
-                if lp[i]-lp[i-1] > 10:
-                    s = i
-                    lp = lp[:s]
-                    break
-            
-            # not a real part
-            if s < 3: 
+            # Split part into subparts >8 lines apart
+            # Discard tiny subparts
+            subparts = mod.strip_and_split(lp, lat_dists, 8)
+            if len(subparts) == 0:
                 continue
+            lp = subparts[0]
+            s = len(lp)
+            
 
             # --- max clique method ----
             G = nx.Graph()

data/ot_events.csv

@@ -5,7 +5,7 @@
 20230113,55624,3,0.08864807555517831,18,"[32, 33, 34, 35, 36, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230114,42503,0,0.32986111111111116,10,"[10, 11, 12, 13, 14, 15, 16, 17, 18, 19]"
 20230114,43050,2,0.07986111111111113,10,"[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"
-20230114,45394,8,0.04241219008264457,6,"[10, 11, 12, 13, 14, 16]"
+20230114,45394,8,0.042412190082644574,6,"[10, 11, 12, 13, 14, 16]"
 20230114,46780,6,0.00994863681649009,6,"[4, 5, 6, 7, 9, 10]"
 20230114,47092,1,0.010377775458032618,7,"[6, 7, 8, 9, 10, 11, 12]"
 20230114,47921,5,0.009933527315815748,7,"[5, 6, 7, 8, 9, 10, 11]"
@@ -15,10 +15,8 @@
 20230114,51009,4,0.2831647828673408,8,"[4, 5, 6, 7, 8, 9, 10, 11]"
 20230114,52396,5,0.02043713781809014,9,"[2, 3, 4, 5, 7, 8, 9, 10, 11]"
 20230114,55491,1,0.034071863389637504,12,"[0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]"
-20230114,56395,5,0.0026803614896185645,4,"[0, 1, 2, 3]"
 20230114,57367,1,0.0168055156564206,8,"[10, 11, 12, 13, 14, 15, 16, 18]"
 20230114,57423,2,0.05307500423992316,8,"[6, 7, 8, 9, 10, 11, 12, 13]"
-20230114,59001,8,0.0028595225001456573,3,"[9, 11, 8]"
 20230114,59327,1,0.0209976353531177,9,"[6, 7, 8, 9, 10, 11, 12, 14, 15]"
 20230114,119106,0,0.033323757540936576,12,"[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20]"
 20230114,133217,7,0.06768516927591008,15,"[32, 34, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31]"
@@ -40,7 +38,7 @@
 20230114,304073,3,0.01409798024266698,8,"[17, 18, 19, 20, 21, 22, 23, 24]"
 20230114,305956,7,0.013200819361201755,7,"[18, 19, 20, 21, 22, 23, 24]"
 20230114,330332,3,0.09014322544669755,18,"[6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
-20230114,333119,14,0.015083635105541806,6,"[15, 20, 21, 22, 23, 26]"
+20230114,333119,14,0.015083635105541804,6,"[15, 20, 21, 22, 23, 26]"
 20230114,345470,7,0.017627784675118715,8,"[18, 19, 20, 21, 22, 23, 24, 25]"
 20230114,364592,3,0.009933527315815748,7,"[17, 18, 20, 21, 22, 23, 24]"
 20230114,369679,3,0.0047550847372912364,5,"[15, 16, 18, 19, 20]"
@@ -60,15 +58,14 @@
 20230114,454762,2,0.00758812296947961,6,"[33, 27, 28, 29, 30, 31]"
 20230114,454972,0,0.004459286539121752,5,"[21, 22, 23, 24, 25]"
 20230114,455084,0,0.013200737298705308,8,"[18, 19, 20, 21, 22, 23, 24, 25]"
-20230114,455603,5,0.0026803614896185645,4,"[27, 28, 29, 31]"
 20230114,458729,3,0.018052664360995934,9,"[10, 11, 12, 13, 15, 16, 17, 18, 19]"
 20230114,459750,0,0.017277305499784082,9,"[23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230114,461497,2,0.007115658806710041,6,"[20, 21, 22, 23, 24, 25]"
 20230114,462407,1,0.015387389327546843,8,"[36, 37, 38, 39, 41, 42, 43, 40]"
 20230114,462775,2,0.01854934872559289,9,"[21, 22, 23, 24, 25, 26, 27, 28, 29]"
 20230114,489140,10,0.5818779041820221,22,"[19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40]"
 20230119,22780,1,0.10699933169971042,20,"[7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28]"
-20230119,23665,8,0.0585509108317194,12,"[11, 12, 13, 14, 15, 18, 20, 21, 22, 23, 26, 19]"
+20230119,23665,8,0.05855091083171939,12,"[11, 12, 13, 14, 15, 18, 20, 21, 22, 23, 26, 19]"
 20230119,29997,2,0.049586207160664975,14,"[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230119,39423,0,0.013200737298705308,8,"[18, 19, 20, 21, 22, 23, 24, 25]"
 20230119,40160,1,0.04091626660918185,13,"[32, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
@@ -88,7 +85,7 @@
 20230126,31082,7,0.13177904727227616,20,"[13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33]"
 20230126,33053,3,0.059417373888560096,15,"[32, 33, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31]"
 20230126,35226,2,0.049586207160664975,14,"[11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]"
-20230126,37224,3,0.029725111699717074,11,"[19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 27]"
+20230126,37224,3,0.029725111699717074,11,"[19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]"
 20230126,38210,3,0.018052664360995934,9,"[13, 14, 16, 17, 19, 20, 21, 22, 23]"
 20230126,41901,3,0.01845993595926091,9,"[32, 33, 25, 26, 27, 28, 29, 30, 31]"
 20230126,57723,20,0.18183045350029725,13,"[32, 33, 35, 36, 37, 20, 21, 22, 25, 26, 27, 29, 31]"
@@ -114,12 +111,11 @@
 20230219,177069,3,0.06731483290248089,16,"[14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]"
 20230219,217710,3,0.08678315712793183,15,"[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230219,219053,2,0.033323757540936576,12,"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12]"
-20230219,219442,14,0.2261789725862624,25,"[7, 8, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34]"
-20230219,228551,3,0.0012870007533763478,3,"[0, 1, 2]"
+20230219,219442,14,0.22617897258626238,25,"[7, 8, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34]"
 20230219,275764,4,0.027927356771982822,11,"[10, 11, 12, 13, 14, 15, 16, 17, 18, 22, 23]"
 20230219,365392,1,0.04001409416307887,13,"[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]"
 20230219,384825,4,0.0737483834664037,16,"[0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17]"
-20230219,385454,8,0.039914201930021556,12,"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12]"
+20230219,385454,8,0.03991420193002156,12,"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12]"
 20230305,49481,2,0.029896810256348653,11,"[12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 15]"
 20230305,141713,0,0.02731716300519501,11,"[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24]"
 20230305,143211,2,0.14258633880580296,22,"[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 35, 36, 39, 32]"
@@ -181,7 +177,6 @@
 20230316,91611,1,0.027927356771982822,11,"[12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]"
 20230316,94072,6,0.02139612638303573,9,"[13, 14, 15, 16, 17, 18, 19, 20, 21]"
 20230316,95574,3,0.010869146299478794,7,"[16, 17, 18, 19, 20, 21, 22]"
-20230316,111937,11,0.008614545350874374,3,"[17, 14, 15]"
 20230318,37565,1,0.07390430340609427,17,"[32, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230318,37627,0,0.16138962431288023,15,"[32, 33, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230318,37686,6,0.0509325541757974,8,"[8, 11, 12, 15, 17, 18, 19, 20]"
@@ -190,7 +185,6 @@
 20230318,55085,0,0.021975308641975347,10,"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
 20230318,62426,5,0.009114936358403757,6,"[10, 11, 12, 14, 15, 16]"
 20230318,63108,13,0.00637599876578004,4,"[16, 18, 12, 11]"
-20230318,63305,10,0.002491526793987608,3,"[11, 12, 13]"
 20230318,63827,9,0.004474250437370517,5,"[16, 17, 21, 15, 18]"
 20230318,66164,1,0.01765864256625719,9,"[14, 15, 16, 17, 18, 19, 20, 21, 22]"
 20230318,66482,7,0.007274287980771944,6,"[13, 14, 15, 16, 18, 17]"
@@ -210,7 +204,7 @@
 20230318,139243,5,0.07049375371802488,16,"[0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19]"
 20230318,154486,0,0.033323757540936576,12,"[12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
 20230318,208516,0,0.055510204081632764,15,"[10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]"
-20230318,225863,13,0.011092868615234182,6,"[14, 16, 17, 18, 19, 20]"
+20230318,225863,13,0.011092868615234184,6,"[14, 16, 17, 18, 19, 20]"
 20230318,266517,0,0.021975308641975347,10,"[14, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
 20230318,286054,0,0.021975308641975347,10,"[12, 13, 14, 15, 16, 17, 18, 19, 20, 21]"
 20230318,286950,0,0.07390430340609427,17,"[7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24]"
@@ -230,7 +224,6 @@
 20230323,94724,0,0.021975308641975347,10,"[15, 16, 17, 18, 19, 20, 21, 22, 23, 24]"
 20230323,95128,1,0.004459286539121752,5,"[1, 2, 3, 4, 5]"
 20230323,95238,0,0.006816552854718002,6,"[15, 16, 17, 18, 19, 20]"
-20230323,95780,8,0.002737845642442469,4,"[4, 5, 6, 7]"
 20230323,99301,4,0.08148483476686284,17,"[12, 16, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 18]"
 20230323,100816,1,0.19635952581881794,26,"[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 20]"
 20230323,119089,1,0.02246340089031397,10,"[17, 18, 19, 20, 21, 22, 23, 24, 25, 26]"
@@ -239,7 +232,7 @@
 20230323,122171,4,0.03484512063947577,11,"[5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16]"
 20230323,123653,1,0.033323757540936576,12,"[32, 33, 34, 35, 36, 37, 25, 26, 27, 28, 30, 31]"
 20230407,23052,12,0.06570853620015465,15,"[12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 24, 26, 10, 15, 7]"
-20230407,31967,14,0.1309990104514488,19,"[32, 33, 34, 35, 36, 38, 39, 41, 42, 44, 24, 25, 26, 28, 29, 30, 31, 40, 17]"
+20230407,31967,14,0.13099901045144877,19,"[32, 33, 34, 35, 36, 38, 39, 41, 42, 44, 24, 25, 26, 28, 29, 30, 31, 40, 17]"
 20230407,32008,9,0.32998934844508176,12,"[10, 12, 13, 14, 15, 16, 17, 18, 22, 8, 6, 7]"
 20230407,46588,0,0.017277305499784082,9,"[16, 17, 18, 19, 20, 21, 22, 23, 24]"
 20230407,48815,0,0.055510204081632764,15,"[32, 33, 34, 35, 36, 37, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
@@ -250,15 +243,14 @@
 20230418,25880,1,0.013965117252655462,8,"[10, 11, 12, 13, 14, 15, 16, 17]"
 20230418,26848,10,0.018477654252715785,8,"[13, 14, 15, 16, 17, 18, 19, 20]"
 20230418,29890,11,0.09690518084837778,17,"[12, 13, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]"
-20230418,30133,6,0.05369734928785446,3,"[0, 1, 2]"
+20230418,30133,6,0.05369734928785446,13,"[14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]"
 20230418,31301,1,0.009933527315815748,7,"[14, 15, 16, 17, 18, 19, 20]"
 20230418,39755,3,0.1277967615044247,21,"[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37]"
 20230418,42023,6,0.13694286856352483,18,"[32, 33, 35, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
-20230418,45613,13,0.005084771274175949,4,"[24, 21, 22, 23]"
 20230418,51694,16,0.049818403010722614,8,"[12, 14, 15, 16, 17, 18, 19, 20]"
 20230418,56571,10,0.04829181030313469,12,"[12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
 20230418,56661,23,0.11220884075943988,12,"[10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 22, 20]"
-20230418,57137,9,0.0313304393697685,10,"[13, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
+20230418,57137,9,0.031330439369768495,10,"[13, 15, 16, 17, 18, 19, 20, 21, 22, 23]"
 20230418,60392,5,0.02037842247930796,9,"[19, 20, 21, 23, 24, 25, 26, 27, 28]"
 20230418,65104,6,0.03271097699039013,8,"[2, 4, 5, 6, 7, 8, 9, 10]"
 20230418,66164,3,0.029069565406175297,11,"[7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18]"
@@ -268,8 +260,8 @@
 20230418,67788,5,0.024004821297046376,10,"[11, 12, 13, 14, 15, 16, 17, 18, 19, 20]"
 20230418,69216,4,0.014179248571534115,8,"[7, 8, 9, 10, 11, 12, 13, 14]"
 20230418,70171,7,0.03429100056945994,11,"[8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]"
-20230418,70965,3,0.00726600918263576,6,"[15, 16, 17, 18, 19, 20]"
-20230418,73498,13,0.07106836592787039,14,"[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
+20230418,70965,3,0.007266009182635759,6,"[15, 16, 17, 18, 19, 20]"
+20230418,73498,13,0.0710683659278704,14,"[18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]"
 20230418,74679,10,0.018814361290670923,8,"[13, 14, 15, 16, 17, 18, 19, 20]"
 20230418,77984,4,0.3825286592358625,36,"[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]"
 20230418,97217,18,0.5747764167162601,20,"[5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25]"
@@ -303,7 +295,7 @@
 20240309,158806,3,0.03484512063947577,12,"[7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]"
 20240309,161189,0,0.017277305499784082,9,"[13, 14, 15, 16, 17, 18, 19, 20, 21]"
 20240309,161924,3,0.01765864256625719,9,"[2, 3, 5, 6, 7, 8, 9, 10, 11]"
-20240309,194829,19,0.20865251766730772,23,"[2, 5, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 28, 29, 30, 32, 34]"
+20240309,194829,19,0.20865251766730775,23,"[2, 5, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 28, 29, 30, 32, 34]"
 20240311,13699,1,0.02246340089031397,10,"[7, 8, 9, 10, 11, 12, 13, 14, 15, 16]"
 20240311,13802,1,0.02731716300519501,11,"[13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24]"
 20240311,14793,0,0.055510204081632764,15,"[14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]"

src/modularity.py

@@ -136,4 +136,44 @@ def dispersion_score(L):
 
     return disp_score
 
-                
+def strip_and_split(part, lat_dists, threshold):
+    """
+    INPUT:
+    part      - a list representing a part in a partition of vertices (indices corresponding to values in lat_dists
+    lat_dists - a list of lateral distances associated with a certain button press
+    threshold - number of lines between values in part that warrant a split into subparts
+
+    OUTPUT:
+    subparts  - a list of lists, each of which is a subpart of part
+    """
+
+    subparts = []
+    part.sort() # ensure
+
+    # split part into subparts each separated with >10 lines from another one
+    sp = []
+    for i in range(len(part)-1):
+        sp.append(part[i])
+        if i+1 == len(part)-1:
+            if part[i+1]-part[i] > threshold:
+                subparts.append(sp)
+                subparts.append([part[i+1]])
+                sp = []
+            else:
+                sp.append(part[i+1])
+                subparts.append(sp)
+        else:
+            if part[i+1]-part[i] > threshold:
+                subparts.append(sp)
+                sp = []
+
+    # discard subparts of length 4 or less
+    no_tiny_subparts = []
+    for sp in subparts:
+        if len(sp) > 4:
+            no_tiny_subparts.append(sp)
+
+    # print("orig. part:", part)
+    # print("subparts  :", subparts)
+    # print("no_tiny   :", no_tiny_subparts)
+    return no_tiny_subparts