Refactor processors

2025-06-07 13:51:27 +02:00
parent 787df8b838
commit d05dc34e97
14 changed files with 2996 additions and 917 deletions
--- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs
+++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs
@@ -14,6 +14,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
    private readonly AppDbContext _db;
    private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger;

+    // Configuration properties loaded from layer records
+    private int Year { get; set; }
+    private List<Record>? Sources { get; set; }
+    private List<Record>? DynamicCodes { get; set; }
+
    public T3MultiSourceYearSummaryProcessor(
        AppDbContext db,
        ILogger<T3MultiSourceYearSummaryProcessor> logger)
@@ -24,69 +29,180 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor

    public override void Process(Layer processWorker)
    {
-        _logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", 
-            processWorker.Name, processWorker.Id);
-
-        var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
-        var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
-        if (sources!.Count == 0)
+        try
        {
-            throw new Exception("Source record not found");
+            _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
+                ProcessorType, processWorker.Name, processWorker.Id);
+
+            // Load configuration from layer records
+            LoadConfiguration(processWorker);
+
+            // Validate required configuration
+            ValidateConfiguration();
+
+            // Perform the actual processing
+            PerformProcessing(processWorker);
+
+            _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
+                ProcessorType, processWorker.Name);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
+                ProcessorType, processWorker.Name, processWorker.Id);
+            throw;
+        }
+    }
+
+    private void LoadConfiguration(Layer processWorker)
+    {
+        if (processWorker.Records == null)
+        {
+            throw new InvalidOperationException("ProcessWorker has no records");
        }

+        // Load year
+        var yearStr = GetRecordValue(processWorker.Records, "Year");
+        if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
+        {
+            throw new InvalidOperationException("Year record not found or invalid");
+        }
+        Year = year;
+
+        // Load sources
+        Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
+        if (Sources.Count == 0)
+        {
+            throw new InvalidOperationException("Source records not found");
+        }
+
+        // Load dynamic codes
+        DynamicCodes = processWorker.Records
+            .Where(x => x.Code!.Contains("DynamicCode-"))
+            .OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
+            .ToList();
+
+        _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
+            ProcessorType, Year, Sources.Count, DynamicCodes.Count);
+    }
+
+    private void ValidateConfiguration()
+    {
+        var errors = new List<string>();
+
+        if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
+        if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
+
+        if (errors.Any())
+        {
+            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
+        }
+
+        _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
+    }
+
+    private void PerformProcessing(Layer processWorker)
+    {
+        _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources",
+            ProcessorType, Year, Sources!.Count);
+
+        // Get or create processed layer
+        var processedLayer = GetOrCreateProcessedLayer(processWorker);
+
+        // Get data sources
+        var dataSources = GetDataSources();
+
+        // Process records (sum by base codes with validation)
+        var newRecords = ProcessRecords(dataSources);
+
+        // Process dynamic codes if configured
+        if (DynamicCodes != null && DynamicCodes.Count > 0)
+        {
+            var calculatedRecords = ProcessDynamicCodes(newRecords);
+            newRecords.AddRange(calculatedRecords);
+        }
+
+        // Save results
+        SaveProcessedLayer(processedLayer, newRecords);
+
+        _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
+            ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
+    }
+
+    private Layer GetOrCreateProcessedLayer(Layer processWorker)
+    {
        var processedLayer = _db.Layers
-            .Where(x => x.ParentId == processWorker.Id
-                        && !x.IsDeleted && !x.IsCancelled)
+            .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
            .OrderByDescending(x => x.CreatedAt)
            .FirstOrDefault();

-        var isNew = false;
        if (processedLayer == null)
        {
-            isNew = true;
            processedLayer = new Layer
            {
                Id = Guid.NewGuid(),
                Type = LayerType.Processed,
                ParentId = processWorker.Id,
-                Number = _db.Layers.Count() + 1
+                Number = _db.Layers.Count() + 1,
+                CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                CreatedAt = DateTime.UtcNow,
+                ModifiedAt = DateTime.UtcNow
            };
-            processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3";
-            processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3";
+
+            _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
+                ProcessorType, processedLayer.Name);
+        }
+        else
+        {
            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
-            processedLayer.CreatedAt = DateTime.UtcNow;
            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
+                ProcessorType, processedLayer.Name);
        }

-        processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
-        processedLayer.ModifiedAt = DateTime.UtcNow;
+        return processedLayer;
+    }

-        var newRecords = new List<Record>();
-
-        var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3"))
+    private List<Layer> GetDataSources()
+    {
+        var dataSources = Sources!.Select(source => 
+            _db.Layers
+                .Where(x => x.Type == LayerType.Processed && 
+                           !x.IsDeleted && !x.IsCancelled && 
+                           x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3"))
                .Include(x => x.Records)
                .AsNoTracking()
                .FirstOrDefault())
            .OfType<Layer>()
            .ToList();
-        
+
        if (dataSources.Count == 0)
        {
-            throw new Exception("DataSources are empty");
+            throw new InvalidOperationException($"No data sources found for year {Year}");
        }

-        var allRecords = dataSources
-            .SelectMany(x => x.Records!).ToList();
+        _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
+            ProcessorType, dataSources.Count, Year);
+
+        return dataSources;
+    }
+
+    private List<Record> ProcessRecords(List<Layer> dataSources)
+    {
+        var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
        var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
+        var newRecords = new List<Record>();
+
+        _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
+            ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);

        foreach (var baseCode in baseCodes)
        {
-            var codeRecords = allRecords.Where(x =>
-                    x.Code![1..] == baseCode)
-                .ToList();
-            var codeRecordsValidation = allRecords.Where(x =>
-                    x.Code![1..] == baseCode)
-                .ToList();
+            var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
+            
            var processedRecord = new Record
            {
                Id = Guid.NewGuid(),
@@ -94,95 +210,119 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
                CreatedAt = DateTime.UtcNow,
                ModifiedAt = DateTime.UtcNow
            };
+
+            // Validation record for double-checking calculations
            var validationRecord = new Record();
+
+            // Sum values from all sources for positions 1-32 with validation
            for (var i = 1; i < 33; i++)
            {
-                ProcessHelper.SetValue(processedRecord, i,
-                    codeRecords.Sum(x => ProcessHelper.GetValue(x, i)));
+                var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
+                ProcessHelper.SetValue(processedRecord, i, totalValue);

-                ProcessHelper.SetValue(validationRecord, i,
-                    codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i)));
+                // Validation calculation (identical to main calculation)
+                var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
+                ProcessHelper.SetValue(validationRecord, i, validationValue);

-                if (
-                    double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) -
-                                        ProcessHelper.GetValue(validationRecord, i))!) > 0.01)
+                // Validate that both calculations match
+                var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!);
+                if (difference > 0.01)
                {
-                    throw new Exception($"ValidationError: Code {baseCode}, " +
-                                        $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " +
-                                        $"{ProcessHelper.GetValue(validationRecord, i)})");
+                    throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})");
                }
            }
+
            newRecords.Add(processedRecord);
+
+            _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
+                ProcessorType, baseCode, codeRecords.Count);
        }

-        // Dynamic Codes
-        var dynamicCodes = processWorker.Records?
-            .Where(x => x.Code!.Contains("DynamicCode-"))
-            .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
-        
-        if (dynamicCodes != null && dynamicCodes.Count != 0)
-        {
-            foreach (var dynamicCode in dynamicCodes)
-            {
-                try
-                {
-                    if (dynamicCode.Desc1 == null)
-                    {
-                        _logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", 
-                            dynamicCode.Id, processWorker.Name, processWorker.Id);
-                        continue;
-                    }
-                    
-                    var calc = new BaseCalc(dynamicCode.Desc1);
-                    if (!calc.IsFormulaCorrect())
-                    {
-                        _logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", 
-                            calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
-                        continue;
-                    }
+        _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
+            ProcessorType, newRecords.Count);

-                    try
-                    {
-                        newRecords.Add(calc.CalculateT3(newRecords));
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", 
-                            calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
-                    }
-                }
-                catch (Exception e)
+        return newRecords;
+    }
+
+    private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
+    {
+        var calculatedRecords = new List<Record>();
+
+        _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
+            ProcessorType, DynamicCodes!.Count);
+
+        foreach (var dynamicCode in DynamicCodes!)
+        {
+            try
+            {
+                if (string.IsNullOrEmpty(dynamicCode.Desc1))
                {
-                    _logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", 
-                        dynamicCode.Id, processWorker.Name, processWorker.Id);
+                    _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
+                        ProcessorType, dynamicCode.Id);
+                    continue;
                }
+
+                var calc = new BaseCalc(dynamicCode.Desc1);
+                if (!calc.IsFormulaCorrect())
+                {
+                    _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
+                        ProcessorType, calc.Expression, dynamicCode.Id);
+                    continue;
+                }
+
+                var calculatedRecord = calc.CalculateT3(baseRecords);
+                calculatedRecords.Add(calculatedRecord);
+
+                _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
+                    ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
+            }
+            catch (Exception e)
+            {
+                _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
+                    ProcessorType, dynamicCode.Desc1);
            }
        }

-        if (isNew)
+        _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
+            ProcessorType, calculatedRecords.Count);
+
+        return calculatedRecords;
+    }
+
+    private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
+    {
+        var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
+
+        if (!existsInDb)
        {
            _db.Layers.Add(processedLayer);
+            _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
        }
        else
        {
            _db.Layers.Update(processedLayer);
+            _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
        }
-        
+
        SaveRecords(processedLayer.Id, newRecords);
        _db.SaveChanges();

-        _logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", 
-            processWorker.Name, processWorker.Id);
+        _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
+            ProcessorType, newRecords.Count, processedLayer.Id);
    }

    private void SaveRecords(Guid layerId, ICollection<Record> records)
    {
+        // Remove existing records for this layer
        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
        if (toDelete.Count > 0)
        {
            _db.Records.RemoveRange(toDelete);
+            _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
+                ProcessorType, toDelete.Count, layerId);
        }

+        // Add new records
        foreach (var record in records)
        {
            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -193,6 +333,12 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
            _db.Records.Add(record);
        }

-        _logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
+        _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
+            ProcessorType, records.Count, layerId);
+    }
+
+    private string? GetRecordValue(ICollection<Record> records, string code)
+    {
+        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
    }
 }