Updated MATLAB scripts: Added new examples for how to load MF4 (unfinalized, finalized, DBC decoded) via the Vehicle Network Toolbox, including via single files, datastores and tall arrays. Also added example on loading MAT files. Added Python script for automating the export of MF4 files to DBC decoded MF4 and MAT.

Author: Martin

.gitignore

@@ -11,4 +11,7 @@ output/
 LOG2/
 *LIN.dbc
 *NMEA-2000-CSS-Electronics-v*
-*find_data_event.py
+*find_data_event.py
+*CSS-Electronics-J1939-2021-08_v1.0.dbc
+logging.py
+*matlab_raw_to_tallarray.m

examples/data-processing/LOG/00000000/00000048/00000001.MF4

@@ -0,0 +1,68 @@
+VERSION ""
+
+
+NS_ : 
+    NS_DESC_
+    CM_
+    BA_DEF_
+    BA_
+    VAL_
+    CAT_DEF_
+    CAT_
+    FILTER
+    BA_DEF_DEF_
+    EV_DATA_
+    ENVVAR_DATA_
+    SGTYPE_
+    SGTYPE_VAL_
+    BA_DEF_SGTYPE_
+    BA_SGTYPE_
+    SIG_TYPE_REF_
+    VAL_TABLE_
+    SIG_GROUP_
+    SIG_VALTYPE_
+    SIGTYPE_VALTYPE_
+    BO_TX_BU_
+    BA_DEF_REL_
+    BA_REL_
+    BA_DEF_DEF_REL_
+    BU_SG_REL_
+    BU_EV_REL_
+    BU_BO_REL_
+    SG_MUL_VAL_
+
+BS_:
+
+BU_:
+
+
+BO_ 2364540158 EEC1: 8 Vector__XXX
+ SG_ EngineSpeed : 24|16@1+ (0.125,0) [0|8031.875] "rpm" Vector__XXX
+
+BO_ 2566844926 CCVS1: 8 Vector__XXX
+ SG_ WheelBasedVehicleSpeed : 8|16@1+ (0.00390625,0) [0|250.996] "km/h" Vector__XXX
+
+
+CM_ BO_ 2364540158 "Electronic Engine Controller 1";
+CM_ SG_ 2364540158 EngineSpeed "Actual engine speed which is calculated over a minimum crankshaft angle of 720 degrees divided by the number of cylinders.…";
+CM_ BO_ 2566844926 "Cruise Control/Vehicle Speed 1";
+CM_ SG_ 2566844926 WheelBasedVehicleSpeed "Wheel-Based Vehicle Speed: Speed of the vehicle as calculated from wheel or tailshaft speed.";
+BA_DEF_ SG_  "SPN" INT 0 524287;
+BA_DEF_ BO_  "VFrameFormat" ENUM  "StandardCAN","ExtendedCAN","reserved","J1939PG";
+BA_DEF_  "DatabaseVersion" STRING ;
+BA_DEF_  "BusType" STRING ;
+BA_DEF_  "ProtocolType" STRING ;
+BA_DEF_  "DatabaseCompiler" STRING ;
+BA_DEF_DEF_  "SPN" 0;
+BA_DEF_DEF_  "VFrameFormat" "J1939PG";
+BA_DEF_DEF_  "DatabaseVersion" "";
+BA_DEF_DEF_  "BusType" "";
+BA_DEF_DEF_  "ProtocolType" "";
+BA_DEF_DEF_  "DatabaseCompiler" "";
+BA_ "ProtocolType" "J1939";
+BA_ "BusType" "CAN";
+BA_ "DatabaseCompiler" "CSS ELECTRONICS (WWW.CSSELECTRONICS.COM)";
+BA_ "DatabaseVersion" "1.0.0";
+BA_ "VFrameFormat" BO_ 2364540158 3;
+BA_ "SPN" SG_ 2364540158 EngineSpeed 190;
+BA_ "SPN" SG_ 2566844926 WheelBasedVehicleSpeed 84;

examples/other/matlab-basics/00000014.MF4

@@ -0,0 +1,158 @@
+VERSION ""
+
+
+NS_ : 
+	NS_DESC_
+	CM_
+	BA_DEF_
+	BA_
+	VAL_
+	CAT_DEF_
+	CAT_
+	FILTER
+	BA_DEF_DEF_
+	EV_DATA_
+	ENVVAR_DATA_
+	SGTYPE_
+	SGTYPE_VAL_
+	BA_DEF_SGTYPE_
+	BA_SGTYPE_
+	SIG_TYPE_REF_
+	VAL_TABLE_
+	SIG_GROUP_
+	SIG_VALTYPE_
+	SIGTYPE_VALTYPE_
+	BO_TX_BU_
+	BA_DEF_REL_
+	BA_REL_
+	BA_DEF_DEF_REL_
+	BU_SG_REL_
+	BU_EV_REL_
+	BU_BO_REL_
+	SG_MUL_VAL_
+
+BS_:
+
+BU_:
+
+
+BO_ 3 gnss_pos: 8 Vector__XXX
+ SG_ PositionAccuracy : 58|6@1+ (1,0) [0|63] "m" Vector__XXX
+ SG_ Latitude : 1|28@1+ (1e-06,-90) [-90|90] "deg" Vector__XXX
+ SG_ Longitude : 29|29@1+ (1e-06,-180) [-180|180] "deg" Vector__XXX
+ SG_ PositionValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+
+BO_ 2 gnss_time: 6 Vector__XXX
+ SG_ TimeValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ TimeConfirmed : 1|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Epoch : 8|40@1+ (0.001,1577840400) [1577840400|2677352027] "sec" Vector__XXX
+
+BO_ 5 gnss_attitude: 8 Vector__XXX
+ SG_ AttitudeValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Roll : 1|12@1+ (0.1,-180) [-180|180] "deg" Vector__XXX
+ SG_ Pitch : 22|12@1+ (0.1,-90) [-90|90] "deg" Vector__XXX
+ SG_ Heading : 43|12@1+ (0.1,0) [0|360] "deg" Vector__XXX
+ SG_ RollAccuracy : 13|9@1+ (0.1,0) [0|50] "deg" Vector__XXX
+ SG_ PitchAccuracy : 34|9@1+ (0.1,0) [0|50] "deg" Vector__XXX
+ SG_ HeadingAccuracy : 55|9@1+ (0.1,0) [0|50] "deg" Vector__XXX
+
+BO_ 6 gnss_odo: 8 Vector__XXX
+ SG_ DistanceTrip : 1|22@1+ (1,0) [0|4194303] "m" Vector__XXX
+ SG_ DistanceAccuracy : 23|19@1+ (1,0) [0|524287] "m" Vector__XXX
+ SG_ DistanceValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ DistanceTotal : 42|22@1+ (1,0) [0|4194303] "km" Vector__XXX
+
+BO_ 1 gnss_status: 1 Vector__XXX
+ SG_ FixType : 0|3@1+ (1,0) [0|5] "" Vector__XXX
+ SG_ Satellites : 3|5@1+ (1,0) [0|31] "" Vector__XXX
+
+BO_ 4 gnss_altitude: 4 Vector__XXX
+ SG_ AltitudeValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Altitude : 1|18@1+ (0.1,-6000) [-6000|20000] "m" Vector__XXX
+ SG_ AltitudeAccuracy : 19|13@1+ (1,0) [0|8000] "m" Vector__XXX
+
+BO_ 8 gnss_geofence: 2 Vector__XXX
+ SG_ FenceValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ FenceCombined : 1|2@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Fence1 : 8|2@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Fence2 : 10|2@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Fence3 : 12|2@1+ (1,0) [0|1] "" Vector__XXX
+ SG_ Fence4 : 14|2@1+ (1,0) [0|1] "" Vector__XXX
+
+BO_ 7 gnss_speed: 5 Vector__XXX
+ SG_ Speed : 1|20@1+ (0.001,0) [0|1048.575] "m/s" Vector__XXX
+ SG_ SpeedAccuracy : 21|19@1+ (0.001,0) [0|524.287] "m/s" Vector__XXX
+ SG_ SpeedValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+
+BO_ 9 gnss_imu: 8 Vector__XXX
+ SG_ AccelerationX : 1|10@1+ (0.125,-64) [-64|63.875] "m/s^2" Vector__XXX
+ SG_ AccelerationY : 11|10@1+ (0.125,-64) [-64|63.875] "m/s^2" Vector__XXX
+ SG_ AccelerationZ : 21|10@1+ (0.125,-64) [-64|63.875] "m/s^2" Vector__XXX
+ SG_ AngularRateX : 31|11@1+ (0.25,-256) [-256|255.75] "deg/s" Vector__XXX
+ SG_ AngularRateY : 42|11@1+ (0.25,-256) [-256|255.75] "deg/s" Vector__XXX
+ SG_ AngularRateZ : 53|11@1+ (0.25,-256) [-256|255.75] "deg/s" Vector__XXX
+ SG_ ImuValid : 0|1@1+ (1,0) [0|1] "" Vector__XXX
+
+
+
+CM_ BO_ 3 "GNSS position";
+CM_ SG_ 3 PositionAccuracy "Accuracy of position";
+CM_ SG_ 3 Latitude "Latitude";
+CM_ SG_ 3 Longitude "Longitude";
+CM_ SG_ 3 PositionValid "Position validity";
+CM_ BO_ 2 "GNSS time";
+CM_ SG_ 2 TimeValid "Time validity";
+CM_ SG_ 2 TimeConfirmed "Time confirmed";
+CM_ SG_ 2 Epoch "Epoch time";
+CM_ BO_ 5 "GNSS attitude";
+CM_ SG_ 5 AttitudeValid "Attitude validity";
+CM_ SG_ 5 Roll "Vehicle roll";
+CM_ SG_ 5 Pitch "Vehicle pitch";
+CM_ SG_ 5 Heading "Vehicle heading";
+CM_ SG_ 5 RollAccuracy "Vehicle roll accuracy";
+CM_ SG_ 5 PitchAccuracy "Vehicle pitch accuracy";
+CM_ SG_ 5 HeadingAccuracy "Vehicle heading accuracy";
+CM_ BO_ 6 "GNSS odometer";
+CM_ SG_ 6 DistanceTrip "Distance traveled since last reset";
+CM_ SG_ 6 DistanceAccuracy "Distance accuracy (1-sigma)";
+CM_ SG_ 6 DistanceTotal "Distance traveled in total";
+CM_ BO_ 1 "GNSS information";
+CM_ SG_ 1 FixType "Fix type";
+CM_ SG_ 1 Satellites "Number of satellites used";
+CM_ BO_ 4 "GNSS altitude";
+CM_ SG_ 4 AltitudeValid "Altitude validity";
+CM_ SG_ 4 Altitude "Altitude";
+CM_ SG_ 4 AltitudeAccuracy "Accuracy of altitude";
+CM_ BO_ 8 "GNSS geofence(s)";
+CM_ SG_ 8 FenceValid "Geofencing status";
+CM_ SG_ 8 FenceCombined "Combined (logical OR) state of all geofences";
+CM_ SG_ 8 Fence1 "Geofence 1 state";
+CM_ SG_ 8 Fence2 "Geofence 2 state";
+CM_ SG_ 8 Fence3 "Geofence 3 state";
+CM_ SG_ 8 Fence4 "Geofence 4 state";
+CM_ BO_ 7 "GNSS speed";
+CM_ SG_ 7 Speed "Speed";
+CM_ SG_ 7 SpeedAccuracy "Speed accuracy";
+CM_ BO_ 9 "GNSS IMU";
+CM_ SG_ 9 AccelerationX "X acceleration with a resolution of 0.125 m/s^2";
+CM_ SG_ 9 AccelerationY "Y acceleration with a resolution of 0.125 m/s^2";
+CM_ SG_ 9 AccelerationZ "Z acceleration with a resolution of 0.125 m/s^2";
+CM_ SG_ 9 AngularRateX "X angular rate with a resolution of 0.25 deg/s";
+CM_ SG_ 9 AngularRateY "Y angular rate with a resolution of 0.25 deg/s";
+CM_ SG_ 9 AngularRateZ "Z angular rate with a resolution of 0.25 deg/s";
+VAL_ 3 PositionValid 0 "Invalid" 1 "Valid" ;
+VAL_ 2 TimeValid 0 "Invalid" 1 "Valid" ;
+VAL_ 2 TimeConfirmed 0 "Unconfirmed" 1 "Confirmed" ;
+VAL_ 5 AttitudeValid 0 "Invalid" 1 "Valid" ;
+VAL_ 6 DistanceValid 0 "Invalid" 1 "Valid" ;
+VAL_ 1 FixType 0 "No fix" 1 "Dead reckoning only" 2 "2D-fix" 3 "3D-fix" 4 "GNSS + dead reckoning combined" 5 "Time only fix" ;
+VAL_ 4 AltitudeValid 0 "Invalid" 1 "Valid" ;
+VAL_ 8 FenceValid 0 "Invalid" 1 "Valid" ;
+VAL_ 8 FenceCombined 0 "Unknown" 1 "Inside" 2 "Outside" ;
+VAL_ 8 Fence1 0 "Unknown" 1 "Inside" 2 "Outside" ;
+VAL_ 8 Fence2 0 "Unknown" 1 "Inside" 2 "Outside" ;
+VAL_ 8 Fence3 0 "Unknown" 1 "Inside" 2 "Outside" ;
+VAL_ 8 Fence4 0 "Unknown" 1 "Inside" 2 "Outside" ;
+VAL_ 7 SpeedValid 0 "Invalid" 1 "Valid" ;
+VAL_ 9 ImuValid 0 "Invalid" 1 "Valid" ;
+

examples/other/matlab-basics/00000014_fin.MF4

@@ -1,29 +1,48 @@
-% This script uses the MATLAB Vehicle Network Toolbox to load
-% finalized & sorted MF4 files from the CANedge (Firmware 01.04.01+)
-% See also below links:
-% - https://www.mathworks.com/help/vnt/mdf-files.html
-% - https://www.mathworks.com/help/vnt/ug/work-with-unfinalized-and-unsorted-mdf-files.html
-% - https://se.mathworks.com/help/vnt/ug/reading-data-from-mdf-files.html
-% - https://se.mathworks.com/help/vnt/ug/using-mdf-files-via-mdf-datastore.html
-% - https://se.mathworks.com/help/vnt/ug/decoding-can-data-from-mdf-files.html
+clear, clc, close all
 
-% load a 'sorted & finalized' MF4 file (via the mdf2finalized converter)
-m = mdf('00000014_fin.MF4')
+% set index of CAN channel (MATLAB finalization: 8 | mdf2finalized: 1) 
+can_idx = 8;
 
-% or, finalize an unfinalized file via MATLAB and load it (support in MATLAB 2021b+)
-finalizedPath = mdfFinalize("00000014.MF4");
-m = mdf(finalizedPath);
+% ------------------------------------------------------------------------
+% finalize & load MF4 "in place" (overwrites original file)
+% m = mdf(mdfFinalize("LOG/11111111/00000012/00000001.MF4"));
 
-% extract 5 rows into a timetable
-rawTimeTable = read(m, 1, m.ChannelNames{1}, 1, 5)
+% finalize & load MF4 "out of place" (makes a copy of original file)
+try
+    finalizedPath2 = mdfFinalize("LOG/11111111/00000012/00000001.MF4", "LOG/11111111/00000012/00000001_fin.MF4");
+    m = mdf(finalizedPath2);
+catch ME
+    disp(ME.message)
+end
 
-% extract data using DBC file (note: MATLAB sees the CANedge data as CAN FD
-% regardless of whether it's Classical CAN or CAN FD)
-db = canDatabase('my_dbc.dbc');
-msgTimetable    = canFDMessageTimetable(rawTimeTable, db)
+% load an MF4 which has already been 'finalized' via MATLAB or mdf2finalized
+m = mdf("LOG/11111111/00000012/00000001_fin.MF4");
 
-% create a signal time table for a specific CAN message and create a plot
-signalTimetable1 = canSignalTimetable(msgTimetable, "Message1")
-plot(signalTimetable1.Time, signalTimetable1.Signal1)
+% extract CAN data into timetable
+rawTimeTable = read(m,can_idx,m.ChannelNames{can_idx});
 
 
+
+% ------------------------------------------------------------------------
+% decode CAN data using DBC, use absolute date & time and extract a specific message
+canDB = canDatabase('dbc_files/canmod-gps.dbc');
+msgTimetableGPS = canFDMessageTimetable(rawTimeTable, canDB);
+msgTimetableGPS.Time = msgTimetableGPS.Time + m.InitialTimestamp;
+msgSpeed = canSignalTimetable(msgTimetableGPS, "gnss_speed");
+
+% decode J1939 data (first convert data to 'Classical' CAN by removing EDL)
+rawTimeTable = removevars(rawTimeTable, "CAN_DataFrame_EDL");
+
+canDB = canDatabase('dbc_files/CSS-Electronics-SAE-J1939-DEMO.dbc');
+msgTimetableJ1939 = j1939ParameterGroupTimetable(rawTimeTable, canDB);
+msgTimetableJ1939.Time = msgTimetableJ1939.Time + m.InitialTimestamp;
+msgEEC1 = j1939SignalTimetable(msgTimetableJ1939, "ParameterGroups","EEC1");
+
+% plot select decoded signals 
+ax1 = subplot(2, 1, 1);
+plot(msgSpeed.Time, msgSpeed.Speed)
+ylabel("Speed (m/s)")
+ax2 = subplot(2, 1, 2);
+plot(msgEEC1.Time, msgEEC1.EngineSpeed)
+ylabel("Engine Speed (rpm)")
+linkaxes([ax1,ax2],'x');

examples/other/matlab-basics/LOG/11111111/00000012/00000001.MF4

@@ -0,0 +1,42 @@
+clear, clc, close all
+
+% load database
+canDB = canDatabase("dbc_files/canmod-gps.dbc");
+
+% create datastore of finalized MF4 files from local disk
+mds = mdfDatastore("LOG_datastore/3BA199E2/00000164", "IncludeSubfolders", true);
+
+% same principle can be used to load MF4 files from mounted S3 drive
+% mds = mdfDatastore("Z:\3BA199E2","IncludeSubfolders", true);
+
+% preview datastore
+preview(mds);
+
+% ------------------------------------------------------------------------
+% if datastore fits into memory you can simply read all data into timetable 
+rawTimeTable = readall(mds);
+msgTimetable = canFDMessageTimetable(rawTimeTable, canDB);
+msgSpeed = canSignalTimetable(msgTimetable, "gnss_speed");
+subplot(2, 1, 1);
+plot(msgSpeed.Time, msgSpeed.Speed)
+ylabel("Speed (m/s) via readall")
+ 
+% ------------------------------------------------------------------------
+% if datastore is larger than memory, use in-memory chunks
+mds.ReadSize = seconds(300);
+msgSpeed = [];
+i = 1;
+
+while hasdata(mds)
+    % read a chunk, decode it and extract data to separate table
+    rawTimeTable = read(mds);
+    msgTimeTable = canFDMessageTimetable(rawTimeTable, canDB);
+    msgSpeedChunk = canSignalTimetable(msgTimetable, "gnss_speed");    
+    msgSpeed = vertcat(msgSpeed,msgSpeedChunk);
+    fprintf("\nreading chunk %i",i)
+    i = i + 1;
+end
+
+subplot(2, 1, 2);
+plot(msgSpeed.Time, msgSpeed.Speed)
+ylabel("Speed (m/s) via chunks")