add elearning community example

Author: sjefvanleeuwen

cdr-aggregation.md

@@ -117,9 +117,18 @@ stateDiagram-v2
     Received --> Validating: Webhook triggers function
     Validating --> IdempotencyChecking: Validation successful
     Validating --> Rejected: Validation failed
+    
     IdempotencyChecking --> Journaling: New CDR
-    IdempotencyChecking --> Skipped: Duplicate CDR
-    Journaling --> Aggregating: Journal stored
+    IdempotencyChecking --> Skipped: Exact duplicate CDR
+    
+    Journaling --> VersionDetection: Journal stored
+    VersionDetection --> Aggregating: No version change
+    VersionDetection --> PreviousVersionRetrieval: New version detected
+    
+    PreviousVersionRetrieval --> DeltaComputation: Get previous version from blob
+    DeltaComputation --> AggregateRecomputation: Calculate change delta
+    AggregateRecomputation --> Aggregating: Apply delta to aggregates
+    
     Aggregating --> Completed: Aggregate updated
     Aggregating --> Failed: Error updating aggregate
     Failed --> Retry: After delay
@@ -129,7 +138,99 @@ stateDiagram-v2
     Completed --> [*]
 ```
 
-*This state diagram represents the processing flow of a CDR through the system, showing the validation, idempotency check, journaling, and aggregation steps.*
+*This state diagram represents the processing flow of a CDR through the system, showing the validation, idempotency check, journaling, and aggregation steps. The diagram includes an alternative path for version handling when updated CDR versions are received. For clarity, version detection occurs after the journal storage to ensure data is safely stored before any further processing.*
+
+#### Version-Based Delta Computation
+
+For clarity, we've added an alternative flow path to handle cases where an updated version of the same CDR is received. This is an important consideration because TandemDrive may send updated versions of previously submitted CDRs when corrections or adjustments are made to charging sessions.
+
+The system uses Azure Blob Storage's built-in versioning capabilities to manage this scenario efficiently:
+
+1. **Version Detection**: When a CDR with the same ID as an existing record is received, the system compares metadata to determine if it's a new version
+2. **Previous Version Retrieval**: If it's a new version, the system retrieves the previous version from blob storage
+3. **Delta Computation**: The system calculates the differences between versions (e.g., changes in energy, duration, or cost)
+4. **Aggregate Recomputation**: The system applies only the delta changes to the aggregates to avoid double-counting
+
+This approach ensures that aggregates remain accurate when CDR data is updated without having to rebuild the entire aggregate from scratch.
+
+##### Implementation Approach (Pseudocode)
+
+```csharp
+// Pseudocode for version-based delta computation
+public async Task ProcessCDRWithVersioning(CDR newCdrVersion)
+{
+    // Check if previous version exists
+    if (await _blobService.ExistsAsync(newCdrVersion.CdrId))
+    {
+        // Get metadata to determine version
+        var metadata = await _blobService.GetMetadataAsync(newCdrVersion.CdrId);
+        
+        if (IsDifferentVersion(metadata, newCdrVersion))
+        {
+            // Get previous version from blob storage using versioning API
+            var previousVersions = await _blobService.ListBlobVersionsAsync(newCdrVersion.CdrId);
+            var latestPreviousVersion = previousVersions.OrderByDescending(v => v.VersionId).First();
+            var previousCdr = await _blobService.GetBlobContentAsync(newCdrVersion.CdrId, latestPreviousVersion.VersionId);
+            
+            // Deserialize previous version
+            var previousCdrData = JsonSerializer.Deserialize<CDR>(previousCdr);
+            
+            // Calculate delta between versions
+            var energyDelta = newCdrVersion.Energy - previousCdrData.Energy;
+            var durationDelta = newCdrVersion.Duration - previousCdrData.Duration;
+            var costDelta = newCdrVersion.Cost - previousCdrData.Cost;
+            
+            // Get existing aggregate
+            var dateKey = newCdrVersion.Timestamp.ToString("yyyy-MM-dd");
+            var monthKey = newCdrVersion.Timestamp.ToString("yyyy-MM");
+            
+            var dailyAggregate = await _tableStorage.GetAggregateAsync(dateKey, newCdrVersion.EvseId);
+            var monthlyAggregate = await _tableStorage.GetAggregateAsync(monthKey, newCdrVersion.EvseId);
+            
+            // Apply deltas to aggregates
+            dailyAggregate.TotalEnergy += energyDelta;
+            dailyAggregate.TotalDuration += durationDelta;
+            dailyAggregate.TotalCost += costDelta;
+            // SessionCount remains unchanged since it's the same session
+            
+            monthlyAggregate.TotalEnergy += energyDelta;
+            monthlyAggregate.TotalDuration += durationDelta;
+            monthlyAggregate.TotalCost += costDelta;
+            
+            // Store updated aggregates
+            await _tableStorage.UpsertAggregateAsync(dailyAggregate);
+            await _tableStorage.UpsertAggregateAsync(monthlyAggregate);
+            
+            // Store new version in blob with versioning enabled
+            await _blobService.StoreWithVersioningAsync(newCdrVersion);
+            
+            _logger.LogInformation(
+                "Updated CDR version processed. ID: {CdrId}, Energy delta: {EnergyDelta}, Cost delta: {CostDelta}",
+                newCdrVersion.CdrId, energyDelta, costDelta);
+                
+            return;
+        }
+    }
+    
+    // If not a version update, proceed with normal processing flow...
+    await ProcessNormalCDR(newCdrVersion);
+}
+
+private bool IsDifferentVersion(BlobMetadata metadata, CDR newCdr)
+{
+    // Check if this is a different version based on metadata
+    // For example, compare version numbers or timestamps
+    return metadata.ContainsKey("version") && 
+           metadata["version"] != newCdr.Version;
+}
+```
+
+This versioning approach provides several benefits:
+1. **Accuracy**: Ensures aggregates reflect the most current data
+2. **Efficiency**: Avoids reprocessing all historical data when a single record changes
+3. **Auditability**: Preserves all versions of CDRs for compliance and reconciliation
+4. **Cost-Effective**: Leverages built-in Azure Blob Storage versioning rather than custom solutions
+5. **Idempotency**: Maintains the system's idempotent processing characteristics
 
 ### Data Model
 Key entities in the system include:

compare.md

@@ -0,0 +1,56 @@
+# Overview
+NOTE: This process supersedes the ([Greenflux CDRs Import Process](https://dev.azure.com/enecomanagedcloud/eMobility/_wiki/wikis/eMobility.wiki/49305/Greenflux-CDRs-Import-process)).
+
+This system is responsible for importing priced charge sessions (CDRs) from TandemDrive external system. The priced charge sessions support `Dynamic electricity tariffs` - instead of paying a fixed price throughout the annual contract, a customer pays a different rate, which typically varies by the hour. This pricing model helps customers reduce energy costs by charging during cheaper periods. 
+
+This import is essential for quick insights and usage analysis (reporting in the Eneco eMobility Portal). It also supports exporting customer-specific data for invoice validation and transparency.
+
+## Data capacity
+We expect a similar volume of data and measurements. While the data model differs, the core information, such as energy consumption and charging metrics, remains the same. The goal is to maintain continuity in reporting and analytics - the output should be functionally equivalent.
+
+## Architecture
+
+This system is designed as a `Real-time processing of big data in motion` architecture. While the data volume does not strictly qualify as Big Data, it is still big enough to require a scalable approach. 
+
+### Preferred Approach - Azure Storage
+This architecture needs to structure blob containers in partitioned way and transform data to Parquet format and it does require CosmosDB for the read part.
+
+![TandemDrive-CDRs-Import-Simplified.drawio.png](/.attachments/TandemDrive-CDRs-Import-Simplified.drawio-2f97d6a1-43f6-499f-83bb-7027d98c6e71.png)
+
+- TandemDrive sends CDRs to Azure API Management via webhook.
+- APIM pushes the data directly into Azure Blob Storage without a backend.
+- Azure Storage holds the raw data, organized for easy access by date or ID.
+- Azure Synapse runs SQL-like queries on the blob data using serverless pools.
+- Backend APIs invoke queries using Synapse, and sends results to the Web Portal.
+
+Blob container should be organized as shown below:
+```bash
+/cdrs/year=2025/month=04/day=22/locationId=LOC123/evseId=EVSE456/file.parquet
+```
+
+### Alternate Approach - Event Hubs
+The overall design resembles a `Kappa architecture`, focusing on stream processing without the need for separate batch and real-time layers.
+
+![HLA-TandemDrive-CDRs-Import-Process.drawio.png](/.attachments/HLA-TandemDrive-CDRs-Import-Process.drawio-379e5d9d-9f7b-4519-b1d6-2044a18e3844.png)
+
+- The system ingests CDRs via Azure API Management (APIM), which exposes a webhook. APIM sends data directly to Event Hubs, with no backend in between.
+- Event Hubs acts as the streaming buffer, supporting replay/reprocessing (allows to re-compute events when fixing bugs or recomputing aggregations)
+- Event Hubs Capture stores all incoming data in Azure Storage automatically, so no custom logic is needed. By default, it stores data in Avro format, optimized for cost efficient long-term storage.
+- Consumers hosted in the App Service consume the stream events in parallel: one saves transformed lightweight CDRs, another one - computes daily summaries (aggregations).
+
+### How we handle versioning
+
+TandemDrive CDRs support versioning, where each update to a CDR includes a `version` field. When a new version arrives, the pipeline:
+- Overwrites the previous version in the CosmosDb by using cdr id and version.
+- Re-computes the daily summaries that include the updated CDR.
+
+This design ensures correct results even if sessions are updated retroactively.
+
+### How we handle failures
+Failures in ingestion and processing are mitigated through the following strategy:
+- Event replay. Event Hubs retains events for 1-7 days (Standard tier, up to 3 months Premium tier). If consumers fail or a bug is discovered, we can reprocess past data without re-ingestion.
+- Retry mechanism. All consumers implement exponential backoff retries to handle transient errors.
+- Consumers only commit their offset (checkpoint) after the event has been processed successfully. This guarantees that events are retried on restart or crash, and ensures consistency in case of transient failures.
+- Poison messages are not retried indefinitely. After the retry limit we forward the message to a custom Dead Letter Queue on Azure Service Bus.
+- Metrics and alerting. Application Insights tracks all failures, retries, and latency metrics. Alerts are configured to notify engineers in case of consistent processing failures or ingestion gaps.
+

diagrams/elearning-community/c4_component.puml

@@ -0,0 +1,57 @@
+@startuml C4_Component
+!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml
+
+LAYOUT_WITH_LEGEND()
+
+title Component diagram for Learning Service
+
+Container(webApp, "Web Application", "Blazor WASM", "Provides the user interface for all platform functionality")
+Container(graphqlApi, "GraphQL API", "Hot Chocolate, .NET 9", "Provides a unified GraphQL schema")
+ContainerDb(learningDb, "Learning Database", "SQL Server", "Stores learning content, paths, and modules")
+ContainerDb(userDb, "User Database", "SQL Server", "Stores user profiles and progress")
+System_Ext(youtubeSystem, "YouTube", "Video hosting platform")
+System_Ext(calendarSystem, "Calendar Service", "Manages event scheduling")
+Container(eventBus, "Event Bus", "Azure Service Bus", "Handles asynchronous communication between services")
+
+Container_Boundary(learningService, "Learning Service") {
+    Component(learningPathManager, "Learning Path Manager", ".NET 9 Service", "Manages creation and structure of learning paths")
+    Component(moduleManager, "Module Manager", ".NET 9 Service", "Handles individual modules within learning paths")
+    Component(contentManager, "Content Manager", ".NET 9 Service", "Manages educational content and resources")
+    Component(calendarIntegration, "Calendar Integration", ".NET 9 Service", "Integrates with calendar services")
+    Component(youtubeIntegration, "YouTube Integration", ".NET 9 Service", "Handles YouTube video embedding")
+    Component(eventManager, "Event Manager", ".NET 9 Service", "Manages learning events and schedules")
+    Component(enrollmentManager, "Enrollment Manager", ".NET 9 Service", "Handles user enrollment in learning paths")
+    Component(progressTracker, "Progress Tracker", ".NET 9 Service", "Tracks user progress through learning paths")
+    Component(learningRepository, "Learning Repository", "EF Core", "Provides data access for learning content")
+}
+
+Rel(graphqlApi, learningPathManager, "Sends learning path operations to")
+Rel(graphqlApi, moduleManager, "Sends module operations to")
+Rel(graphqlApi, contentManager, "Sends content operations to")
+Rel(graphqlApi, eventManager, "Sends event operations to")
+Rel(graphqlApi, enrollmentManager, "Sends enrollment operations to")
+Rel(graphqlApi, progressTracker, "Sends progress queries to")
+
+Rel(learningPathManager, moduleManager, "Uses")
+Rel(moduleManager, contentManager, "Uses")
+Rel(eventManager, calendarIntegration, "Uses")
+Rel(contentManager, youtubeIntegration, "Uses")
+
+Rel(learningPathManager, learningRepository, "Uses")
+Rel(moduleManager, learningRepository, "Uses")
+Rel(contentManager, learningRepository, "Uses")
+Rel(eventManager, learningRepository, "Uses")
+Rel(enrollmentManager, learningRepository, "Uses")
+Rel(progressTracker, learningRepository, "Uses")
+
+Rel(learningRepository, learningDb, "Reads from and writes to")
+Rel(enrollmentManager, userDb, "Reads from and writes to")
+Rel(progressTracker, userDb, "Reads from and writes to")
+
+Rel(youtubeIntegration, youtubeSystem, "Embeds videos from", "HTTPS")
+Rel(calendarIntegration, calendarSystem, "Schedules events using", "HTTPS")
+
+Rel(eventManager, eventBus, "Publishes learning events to")
+Rel(enrollmentManager, eventBus, "Publishes enrollment events to")
+
+@enduml

diagrams/elearning-community/c4_container.puml

@@ -0,0 +1,70 @@
+@startuml C4_Container
+!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml
+
+LAYOUT_WITH_LEGEND()
+
+title Container diagram for E-Learning Community Platform
+
+Person(contentCreator, "Content Creator", "Creates learning paths, modules, and assigns reviewers")
+Person(learner, "Learner", "Follows learning paths, completes assignments, earns certifications")
+Person(reviewer, "Reviewer", "Reviews and grades assignments, provides feedback")
+Person(admin, "Administrator", "Manages platform settings and user access")
+
+System_Ext(youtubeSystem, "YouTube", "Video hosting platform")
+System_Ext(calendarSystem, "Calendar Service", "Manages event scheduling")
+System_Ext(emailSystem, "Email Service", "Sends notifications and alerts")
+System_Ext(authSystem, "Authentication Provider", "Identity management")
+
+System_Boundary(elearningSystem, "E-Learning Community Platform") {
+    Container(webApp, "Web Application", "Blazor WASM, .NET 9", "Provides the user interface for all platform functionality")
+    
+    Container(apiGateway, "API Gateway", "Azure API Management", "Routes requests and handles API management")
+    
+    Container(graphqlApi, "GraphQL API", "Hot Chocolate, .NET 9", "Provides a unified GraphQL schema for the web application")
+    
+    Container(learningService, "Learning Service", ".NET 9", "Manages learning paths, modules, and content")
+    Container(assignmentService, "Assignment Service", ".NET 9", "Handles assignment submissions and reviews")
+    Container(forumService, "Forum Service", ".NET 9", "Provides discussion forum capabilities")
+    Container(certificationService, "Certification Service", ".NET 9", "Tracks progress and issues certifications")
+    Container(notificationService, "Notification Service", ".NET 9", "Manages notifications and alerts")
+    
+    ContainerDb(learningDb, "Learning Database", "SQL Server", "Stores learning content, paths, and modules")
+    ContainerDb(assignmentDb, "Assignment Database", "SQL Server", "Stores assignments, submissions, and grades")
+    ContainerDb(forumDb, "Forum Database", "SQL Server", "Stores forum posts and discussions")
+    ContainerDb(userDb, "User Database", "SQL Server", "Stores user profiles and progress")
+    
+    Container(fileStorage, "File Storage", "Azure Blob Storage", "Stores assignment files and resources")
+    Container(eventBus, "Event Bus", "Azure Service Bus", "Handles asynchronous communication between services")
+}
+
+Rel(contentCreator, webApp, "Uses", "HTTPS")
+Rel(learner, webApp, "Uses", "HTTPS")
+Rel(reviewer, webApp, "Uses", "HTTPS")
+Rel(admin, webApp, "Uses", "HTTPS")
+
+Rel(webApp, graphqlApi, "Makes API calls to", "HTTPS/GraphQL")
+Rel(graphqlApi, apiGateway, "Routes requests through")
+
+Rel(apiGateway, learningService, "Routes learning requests to")
+Rel(apiGateway, assignmentService, "Routes assignment requests to")
+Rel(apiGateway, forumService, "Routes forum requests to")
+Rel(apiGateway, certificationService, "Routes certification requests to")
+Rel(apiGateway, notificationService, "Routes notification requests to")
+
+Rel(learningService, learningDb, "Reads from and writes to")
+Rel(assignmentService, assignmentDb, "Reads from and writes to")
+Rel(assignmentService, fileStorage, "Stores files in")
+Rel(forumService, forumDb, "Reads from and writes to")
+Rel(certificationService, learningDb, "Reads from")
+Rel(certificationService, assignmentDb, "Reads from")
+Rel(certificationService, userDb, "Reads from and writes to")
+Rel(notificationService, eventBus, "Publishes events to")
+
+Rel(learningService, youtubeSystem, "Embeds videos from", "HTTPS")
+Rel(learningService, calendarSystem, "Schedules events using", "HTTPS")
+Rel(notificationService, emailSystem, "Sends emails through", "SMTP")
+Rel(webApp, authSystem, "Authenticates using", "OAuth 2.0")
+
+Rel(eventBus, notificationService, "Triggers notifications in")
+
+@enduml

diagrams/elearning-community/c4_context.puml

@@ -0,0 +1,34 @@
+@startuml C4_Context
+!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Context.puml
+
+LAYOUT_WITH_LEGEND()
+
+title System Context diagram for E-Learning Community Platform
+
+Person(contentCreator, "Content Creator", "Creates learning paths, modules, and assigns reviewers")
+Person(learner, "Learner", "Follows learning paths, completes assignments, earns certifications")
+Person(reviewer, "Reviewer", "Reviews and grades assignments, provides feedback")
+Person(admin, "Administrator", "Manages platform settings and user access")
+
+System(elearningSystem, "E-Learning Community Platform", "Enables creation and consumption of educational content, assignments, and certification")
+
+System_Ext(youtubeSystem, "YouTube", "Video hosting platform")
+System_Ext(calendarSystem, "Calendar Service", "Manages event scheduling")
+System_Ext(emailSystem, "Email Service", "Sends notifications and alerts")
+System_Ext(authSystem, "Authentication Provider", "Identity management")
+
+Rel(contentCreator, elearningSystem, "Creates learning content and paths")
+Rel(learner, elearningSystem, "Follows learning paths, submits assignments")
+Rel(reviewer, elearningSystem, "Reviews and grades assignments")
+Rel(admin, elearningSystem, "Manages platform and users")
+
+Rel(elearningSystem, youtubeSystem, "Embeds videos from")
+Rel(elearningSystem, calendarSystem, "Schedules events using")
+Rel(elearningSystem, emailSystem, "Sends notifications through")
+Rel(elearningSystem, authSystem, "Authenticates users via")
+
+Rel(emailSystem, learner, "Sends notifications to")
+Rel(emailSystem, contentCreator, "Sends notifications to")
+Rel(emailSystem, reviewer, "Sends notifications to")
+
+@enduml