Minor fix to can_decoder. Added support for optional signal prefixes via inputs.py file

Author: MatinF

deploy-aws-lambda/arn-layers/lambda_layer_arns.csv

@@ -1,17 +1,17 @@
-ap-northeast-1,arn:aws:lambda:ap-northeast-1:319723967016:layer:canedge-influxdb-writer-2:8
-ap-northeast-2,arn:aws:lambda:ap-northeast-2:319723967016:layer:canedge-influxdb-writer-2:8
-ap-northeast-3,arn:aws:lambda:ap-northeast-3:319723967016:layer:canedge-influxdb-writer-2:8
-ap-south-1,arn:aws:lambda:ap-south-1:319723967016:layer:canedge-influxdb-writer-2:8
-ap-southeast-1,arn:aws:lambda:ap-southeast-1:319723967016:layer:canedge-influxdb-writer-2:8
-ap-southeast-2,arn:aws:lambda:ap-southeast-2:319723967016:layer:canedge-influxdb-writer-2:8
-ca-central-1,arn:aws:lambda:ca-central-1:319723967016:layer:canedge-influxdb-writer-2:8
-eu-central-1,arn:aws:lambda:eu-central-1:319723967016:layer:canedge-influxdb-writer-2:10
-eu-north-1,arn:aws:lambda:eu-north-1:319723967016:layer:canedge-influxdb-writer-2:10
-eu-west-1,arn:aws:lambda:eu-west-1:319723967016:layer:canedge-influxdb-writer-2:8
-eu-west-2,arn:aws:lambda:eu-west-2:319723967016:layer:canedge-influxdb-writer-2:8
-eu-west-3,arn:aws:lambda:eu-west-3:319723967016:layer:canedge-influxdb-writer-2:8
-sa-east-1,arn:aws:lambda:sa-east-1:319723967016:layer:canedge-influxdb-writer-2:8
-us-east-1,arn:aws:lambda:us-east-1:319723967016:layer:canedge-influxdb-writer-2:8
-us-east-2,arn:aws:lambda:us-east-2:319723967016:layer:canedge-influxdb-writer-2:8
-us-west-1,arn:aws:lambda:us-west-1:319723967016:layer:canedge-influxdb-writer-2:8
-us-west-2,arn:aws:lambda:us-west-2:319723967016:layer:canedge-influxdb-writer-2:8
+ap-northeast-1,arn:aws:lambda:ap-northeast-1:319723967016:layer:canedge-influxdb-writer-2:9
+ap-northeast-2,arn:aws:lambda:ap-northeast-2:319723967016:layer:canedge-influxdb-writer-2:9
+ap-northeast-3,arn:aws:lambda:ap-northeast-3:319723967016:layer:canedge-influxdb-writer-2:9
+ap-south-1,arn:aws:lambda:ap-south-1:319723967016:layer:canedge-influxdb-writer-2:9
+ap-southeast-1,arn:aws:lambda:ap-southeast-1:319723967016:layer:canedge-influxdb-writer-2:9
+ap-southeast-2,arn:aws:lambda:ap-southeast-2:319723967016:layer:canedge-influxdb-writer-2:9
+ca-central-1,arn:aws:lambda:ca-central-1:319723967016:layer:canedge-influxdb-writer-2:9
+eu-central-1,arn:aws:lambda:eu-central-1:319723967016:layer:canedge-influxdb-writer-2:11
+eu-north-1,arn:aws:lambda:eu-north-1:319723967016:layer:canedge-influxdb-writer-2:11
+eu-west-1,arn:aws:lambda:eu-west-1:319723967016:layer:canedge-influxdb-writer-2:9
+eu-west-2,arn:aws:lambda:eu-west-2:319723967016:layer:canedge-influxdb-writer-2:9
+eu-west-3,arn:aws:lambda:eu-west-3:319723967016:layer:canedge-influxdb-writer-2:9
+sa-east-1,arn:aws:lambda:sa-east-1:319723967016:layer:canedge-influxdb-writer-2:9
+us-east-1,arn:aws:lambda:us-east-1:319723967016:layer:canedge-influxdb-writer-2:9
+us-east-2,arn:aws:lambda:us-east-2:319723967016:layer:canedge-influxdb-writer-2:9
+us-west-1,arn:aws:lambda:us-west-1:319723967016:layer:canedge-influxdb-writer-2:9
+us-west-2,arn:aws:lambda:us-west-2:319723967016:layer:canedge-influxdb-writer-2:9

deploy-aws-lambda/arn-layers/requirements.txt

@@ -5,7 +5,7 @@ aiosignal==1.2.0
 async-timeout==4.0.2
 attrs==21.4.0
 botocore==1.23.24
-can-decoder>=0.1.8
+can-decoder>=0.1.9
 canedge-browser>=0.0.8
 canmatrix==0.9.5
 certifi==2021.10.8

inputs.py

@@ -11,8 +11,12 @@
 
 # -----------------------------------------------
 # specify DBC paths and a list of signals to process ([]: include all signals)
+# optionally include signal prefixes to make CAN ID, PGN and/or BusChannel explicit
 dbc_paths = ["dbc_files/canmod-gps.dbc"]
 signals = []
+can_id_prefix = False
+pgn_prefix = False
+bus_prefix = False
 
 # specify resampling frequency. Setting this to "" means no resampling (much slower)
 res = "5S"

main.py

@@ -4,7 +4,8 @@
     list_log_files,
     ProcessData,
     MultiFrameDecoder,
-    restructure_data
+    restructure_data,
+    add_signal_prefix
 )
 from utils_db import SetupInflux
 import inputs as inp
@@ -30,6 +31,8 @@
 
     df_phys = proc.extract_phys(df_raw)
     proc.print_log_summary(device_id, log_file, df_phys)
+    
+    df_phys = add_signal_prefix(df_phys, can_id_prefix=inp.can_id_prefix, pgn_prefix=inp.pgn_prefix, bus_prefix=inp.bus_prefix)
 
     df_phys = restructure_data(df_phys,inp.res)
     influx.write_signals(device_id, df_phys)

requirements.txt

@@ -5,7 +5,7 @@ aiosignal==1.2.0
 async-timeout==4.0.2
 attrs==21.4.0
 botocore==1.23.24
-can-decoder>=0.1.8
+can-decoder>=0.1.9
 canedge-browser>=0.0.8
 canmatrix==0.9.5
 certifi==2021.10.8

utils.py

@@ -73,19 +73,22 @@ def list_log_files(fs, devices, start_times, verbose=True, passwords={}):
 
     return log_files
 
-def add_signal_prefix(df_phys, can_id_prefix=False, pgn_prefix=False):
+def add_signal_prefix(df_phys, can_id_prefix=False, pgn_prefix=False, bus_prefix=False):
     """Rename Signal names by prefixing the full
     CAN ID (in hex) and/or J1939 PGN
     """
     from J1939_PGN import J1939_PGN
-
-    if can_id_prefix == True and pgn_prefix == False:
-        df_phys["Signal"] = df_phys["CAN ID"].apply(lambda x: f"{hex(int(x))[2:].upper()}") + "." + df_phys["Signal"]
-    elif can_id_prefix == True and pgn_prefix == True:
-        df_phys["Signal"] = df_phys["CAN ID"].apply(lambda x: f"{hex(int(x))[2:].upper()}.{J1939_PGN(int(x)).pgn}") + "." + df_phys["Signal"]
-    elif can_id_prefix == False and pgn_prefix == True:
-        df_phys["Signal"] = df_phys["CAN ID"].apply(lambda x: f"{J1939_PGN(int(x)).pgn}") + "." + df_phys["Signal"]
-
+    
+    prefix = ""
+    if bus_prefix:
+        prefix += df_phys["BusChannel"].apply(lambda x: f"{x}.")
+    if can_id_prefix:
+        prefix += df_phys["CAN ID"].apply(lambda x: f"{hex(int(x))[2:].upper()}." )
+    if pgn_prefix:
+        prefix += df_phys["CAN ID"].apply(lambda x: f"{J1939_PGN(int(x)).pgn}.")
+        
+    df_phys["Signal"] = prefix + df_phys["Signal"]
+    
     return df_phys
 
 def restructure_data(df_phys, res, ffill=False):
@@ -173,11 +176,15 @@ def extract_phys(self, df_raw):
         for db in self.db_list:
             df_decoder = can_decoder.DataFrameDecoder(db)
 
-            for length, group in df_raw.groupby("DataLength"):
-                df_phys_temp.append(df_decoder.decode_frame(group))
-
+            for bus, bus_group in df_raw.groupby("BusChannel"):  
+                for length, group in bus_group.groupby("DataLength"):
+                    df_phys_group = df_decoder.decode_frame(group)
+                    if not df_phys_group.empty:
+                        df_phys_group["BusChannel"] = bus 
+                    df_phys_temp.append(df_phys_group)
+                    
         df_phys = pd.concat(df_phys_temp, ignore_index=False).sort_index()
-
+        
         # remove duplicates in case multiple DBC files contain identical signals
         df_phys["datetime"] = df_phys.index
         df_phys = df_phys.drop_duplicates(keep="first")