Morska.Process.D6

2025-06-16 20:36:48 +02:00
parent 4c48484e2f
commit bbb3f5b2bf
4 changed files with 436 additions and 3 deletions
--- a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs
+++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs
@@ -0,0 +1,377 @@
+using System.Globalization;
+using System.Text.RegularExpressions;
+using DiunaBI.Core.Services;
+using Google.Apis.Sheets.v4;
+using Google.Apis.Sheets.v4.Data;
+using Microsoft.EntityFrameworkCore;
+using DiunaBI.Core.Models;
+using DiunaBI.Core.Database.Context;
+using Microsoft.Extensions.Logging;
+
+namespace DiunaBI.Plugins.Morska.Processors;
+
+public class MorskaD6Processor : MorskaBaseProcessor
+{
+    public override string ProcessorType => "Morska.Process.D6";
+
+    private readonly AppDbContext _db;
+    private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
+    private readonly ILogger<MorskaT1R3Processor> _logger;
+
+    // Configuration properties loaded from layer records
+    private int Year { get; set; }
+    private string? CostSource { get; set; }
+    private string? SellSource { get; set; }
+
+    public MorskaD6Processor(
+        AppDbContext db,
+        SpreadsheetsResource.ValuesResource googleSheetValues,
+        ILogger<MorskaT1R3Processor> logger)
+    {
+        _db = db;
+        _googleSheetValues = googleSheetValues;
+        _logger = logger;
+    }
+
+    public override void Process(Layer processWorker)
+    {
+        try
+        {
+            _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
+                ProcessorType, processWorker.Name, processWorker.Id);
+
+            // Load configuration from layer records
+            LoadConfiguration(processWorker);
+
+            // Validate required configuration
+            ValidateConfiguration();
+
+            // Perform the actual processing
+            PerformProcessing(processWorker);
+
+            _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
+                ProcessorType, processWorker.Name);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
+                ProcessorType, processWorker.Name, processWorker.Id);
+            throw;
+        }
+    }
+
+    private void LoadConfiguration(Layer processWorker)
+    {
+        if (processWorker.Records == null)
+        {
+            throw new InvalidOperationException("ProcessWorker has no records");
+        }
+
+        var yearStr = GetRecordValue(processWorker.Records, "Year");
+        if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
+        {
+            throw new InvalidOperationException("Year record not found or invalid");
+        }
+
+        Year = year;
+
+        CostSource = GetRecordValue(processWorker.Records, "SourceLayerCost");
+        if (string.IsNullOrEmpty(CostSource))
+        {
+            throw new InvalidOperationException("SourceLayerCost record not found");
+        }
+
+        SellSource = GetRecordValue(processWorker.Records, "SourceLayerSell");
+        if (string.IsNullOrEmpty(SellSource))
+        {
+            throw new InvalidOperationException("SourceLayerCosts record not found");
+        }
+
+        _logger.LogDebug(
+            "{ProcessorType}: Configuration loaded - Year: {Year}, SourceCost: {CostSource}, SourceSell: {SellSource}",
+            ProcessorType, Year, CostSource, SellSource);
+    }
+
+    private void ValidateConfiguration()
+    {
+        var errors = new List<string>();
+
+        if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
+        if (string.IsNullOrEmpty(CostSource)) errors.Add("CostSource is required");
+        if (string.IsNullOrEmpty(SellSource)) errors.Add("SellSource is required");
+
+        if (errors.Any())
+        {
+            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
+        }
+
+        _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
+    }
+
+    private void PerformProcessing(Layer processWorker)
+    {
+        _logger.LogDebug(
+            "{ProcessorType}: Processing data for Year: {Year}, CostSource: {CostSource}, SellSource: {SellSource}",
+            ProcessorType, Year, CostSource, SellSource);
+
+        var processedLayer = GetOrCreateProcessedLayer(processWorker);
+
+        var dataSources = GetDataSources();
+
+        var newRecords = ProcessRecords(dataSources);
+
+        SaveProcessedLayer(processedLayer, newRecords);
+
+        _logger.LogInformation(
+            "{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
+            ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
+    }
+
+    private Layer GetOrCreateProcessedLayer(Layer processWorker)
+    {
+        var processedLayer = _db.Layers
+            .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
+            .OrderByDescending(x => x.CreatedAt)
+            .FirstOrDefault();
+
+        if (processedLayer == null)
+        {
+            processedLayer = new Layer
+            {
+                Id = Guid.NewGuid(),
+                Type = LayerType.Processed,
+                ParentId = processWorker.Id,
+                Number = _db.Layers.Count() + 1,
+                CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                CreatedAt = DateTime.UtcNow,
+                ModifiedAt = DateTime.UtcNow
+            };
+            processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-D6";
+
+            _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
+                ProcessorType, processedLayer.Name);
+        }
+        else
+        {
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
+                ProcessorType, processedLayer.Name);
+        }
+
+        return processedLayer;
+    }
+
+    private List<Layer> GetDataSources()
+    {
+        var costDataSource = _db.Layers
+            .Include(layer => layer.Records!)
+            .AsNoTracking()
+            .FirstOrDefault(x => x.Name == CostSource && !x.IsDeleted && !x.IsCancelled);
+
+        if (costDataSource == null)
+        {
+            throw new InvalidOperationException($"CostDataSource not found {CostSource}");
+        }
+
+        var sellDataSource = _db.Layers
+            .Include(layer => layer.Records!)
+            .AsNoTracking()
+            .FirstOrDefault(x => x.Name == SellSource && !x.IsDeleted && !x.IsCancelled);
+
+        if (sellDataSource == null)
+        {
+            throw new InvalidOperationException($"SellDataSource not found {SellSource}");
+        }
+
+        _logger.LogDebug("{ProcessorType}: Found both data sources data sources: {CostSource}, {SellSource}",
+            ProcessorType, CostSource, SellSource);
+        ;
+
+        return [costDataSource, sellDataSource];
+    }
+
+    private List<Record> ProcessRecords(List<Layer> dataSources)
+    {
+        var newRecords = new List<Record>();
+
+        // L8542-D-DEPARTMENTS
+        var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542);
+        
+        var departmentLookup = new Dictionary<string, string>();
+        if (dictionary?.Records != null)
+        {
+            foreach (var dictRecord in dictionary.Records)
+            {
+                if (!string.IsNullOrEmpty(dictRecord.Desc1) && !string.IsNullOrEmpty(dictRecord.Code))
+                {
+                    departmentLookup[dictRecord.Desc1] = dictRecord.Code;
+                }
+            }
+
+            _logger.LogDebug("{ProcessorType}: Loaded {DictCount} department mappings from dictionary",
+                ProcessorType, departmentLookup.Count);
+        }
+        else
+        {
+            _logger.LogWarning("{ProcessorType}: Department dictionary (layer 8542) not found or has no records",
+                ProcessorType);
+        }
+
+        var firstDataSource = dataSources.First();
+
+        if (firstDataSource.Records == null || !firstDataSource.Records.Any())
+        {
+            _logger.LogWarning("{ProcessorType}: First data source has no records to process", ProcessorType);
+            return newRecords;
+        }
+
+        var groupedData = firstDataSource.Records
+            .Where(record => record.Code != null && record.Code.Length >= 8 && record.Value1.HasValue)
+            .Select(record => new
+            {
+                Month = record.Code!.Substring(4, 2),
+                Type = record.Code.Substring(6, 2),
+                OriginalDepartment = record.Desc1 ?? string.Empty,
+                Value = record.Value1!.Value,
+                Department = GetDepartmentByType(record.Code.Substring(6, 2), record.Desc1 ?? string.Empty)
+            })
+            .Select(x => new
+            {
+                x.Month,
+                x.Type,
+                x.Value,
+                x.Department,
+                DepartmentCode = departmentLookup.TryGetValue(x.Department, out var value1)
+                    ? value1
+                    : x.Department,
+                FinalCode =
+                    $"2{(departmentLookup.TryGetValue(x.Department, out var value) ? value : x.Department)}{x.Type}{x.Month}"
+            })
+            .GroupBy(x => x.FinalCode)
+            .Select(group => new
+            {
+                FinalCode = group.Key,
+                Department = group.First().Department,
+                DepartmentCode = group.First().DepartmentCode,
+                Type = group.First().Type,
+                Month = group.First().Month,
+                TotalValue = group.Sum(x => x.Value)
+            })
+            .ToList();
+
+        foreach (var groupedRecord in groupedData)
+        {
+            var newRecord = new Record
+            {
+                Id = Guid.NewGuid(),
+                Code = groupedRecord.FinalCode,
+                CreatedAt = DateTime.UtcNow,
+                ModifiedAt = DateTime.UtcNow,
+                Value1 = groupedRecord.TotalValue,
+                Desc1 = groupedRecord.Department
+            };
+
+            newRecords.Add(newRecord);
+
+            _logger.LogDebug(
+                "{ProcessorType}: Created summed record {NewRecordCode} with total value {Value} for department {Department} (code: {DepartmentCode}), type {Type}, month {Month}",
+                ProcessorType, newRecord.Code, newRecord.Value1, groupedRecord.Department, groupedRecord.DepartmentCode,
+                groupedRecord.Type, groupedRecord.Month);
+        }
+
+        _logger.LogInformation(
+            "{ProcessorType}: Processed {GroupCount} unique grouped records from {OriginalCount} original records",
+            ProcessorType, newRecords.Count, firstDataSource.Records.Count);
+
+        return newRecords;
+    }
+
+    private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
+    {
+        var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
+
+        if (!existsInDb)
+        {
+            _db.Layers.Add(processedLayer);
+            _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
+        }
+        else
+        {
+            _db.Layers.Update(processedLayer);
+            _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
+        }
+
+        SaveRecords(processedLayer.Id, newRecords);
+        _db.SaveChanges();
+
+        _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
+            ProcessorType, newRecords.Count, processedLayer.Id);
+    }
+
+    private void SaveRecords(Guid layerId, ICollection<Record> records)
+    {
+        // Remove existing records for this layer
+        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
+        if (toDelete.Count > 0)
+        {
+            _db.Records.RemoveRange(toDelete);
+            _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
+                ProcessorType, toDelete.Count, layerId);
+        }
+
+        // Add new records
+        foreach (var record in records)
+        {
+            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            record.CreatedAt = DateTime.UtcNow;
+            record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            record.ModifiedAt = DateTime.UtcNow;
+            record.LayerId = layerId;
+            _db.Records.Add(record);
+        }
+
+        _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
+            ProcessorType, records.Count, layerId);
+    }
+
+    private string? GetRecordValue(ICollection<Record> records, string code)
+    {
+        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
+    }
+
+    private string GetDepartmentByType(string type, string originalDepartment)
+    {
+        var typesThatUseDepartment = new[] { "02", "09", "10", "11", "12", "13", "14", "15" };
+        var typesThatUseAK = new[] { "03", "06", "07", "08" };
+
+        if (typesThatUseDepartment.Contains(type))
+        {
+            return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
+        }
+
+        if (typesThatUseAK.Contains(type))
+        {
+            return "AK";
+        }
+
+        if (type == "04")
+        {
+            return "PU";
+        }
+
+        if (type == "05")
+        {
+            return "OTHER";
+        }
+
+        {
+            _logger.LogWarning("{ProcessorType}: Unknown type {Type}, using {Department}",
+                nameof(MorskaD6Processor), type,
+                string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment);
+            return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
+        }
+    }
+}
--- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
+++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
@@ -448,7 +448,8 @@ public class LayersController : Controller
            "T5-LastValues",
            "T1-R1",
            "T4-R2",
-            "T1-R3"
+            "T1-R3",
+            "D6"
        ];

        _logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length);
@@ -580,6 +581,16 @@ public class LayersController : Controller
                    processor.Process(processWorker);
                    return;
                }
+            case "D6":
+                {
+                    var processor = _pluginManager.GetProcessor(plugin);
+                    if (processor == null)
+                    {
+                        throw new Exception("D6 processor not found");
+                    }
+                    processor.Process(processWorker);
+                    return;
+                }
        }

        var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1;