diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs new file mode 100644 index 0000000..79791a6 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs @@ -0,0 +1,377 @@ +using System.Globalization; +using System.Text.RegularExpressions; +using DiunaBI.Core.Services; +using Google.Apis.Sheets.v4; +using Google.Apis.Sheets.v4.Data; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaD6Processor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.D6"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? CostSource { get; set; } + private string? SellSource { get; set; } + + public MorskaD6Processor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + + Year = year; + + CostSource = GetRecordValue(processWorker.Records, "SourceLayerCost"); + if (string.IsNullOrEmpty(CostSource)) + { + throw new InvalidOperationException("SourceLayerCost record not found"); + } + + SellSource = GetRecordValue(processWorker.Records, "SourceLayerSell"); + if (string.IsNullOrEmpty(SellSource)) + { + throw new InvalidOperationException("SourceLayerCosts record not found"); + } + + _logger.LogDebug( + "{ProcessorType}: Configuration loaded - Year: {Year}, SourceCost: {CostSource}, SourceSell: {SellSource}", + ProcessorType, Year, CostSource, SellSource); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(CostSource)) errors.Add("CostSource is required"); + if (string.IsNullOrEmpty(SellSource)) errors.Add("SellSource is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug( + "{ProcessorType}: Processing data for Year: {Year}, CostSource: {CostSource}, SellSource: {SellSource}", + ProcessorType, Year, CostSource, SellSource); + + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + var dataSources = GetDataSources(); + + var newRecords = ProcessRecords(dataSources); + + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation( + "{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-D6"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var costDataSource = _db.Layers + .Include(layer => layer.Records!) + .AsNoTracking() + .FirstOrDefault(x => x.Name == CostSource && !x.IsDeleted && !x.IsCancelled); + + if (costDataSource == null) + { + throw new InvalidOperationException($"CostDataSource not found {CostSource}"); + } + + var sellDataSource = _db.Layers + .Include(layer => layer.Records!) + .AsNoTracking() + .FirstOrDefault(x => x.Name == SellSource && !x.IsDeleted && !x.IsCancelled); + + if (sellDataSource == null) + { + throw new InvalidOperationException($"SellDataSource not found {SellSource}"); + } + + _logger.LogDebug("{ProcessorType}: Found both data sources data sources: {CostSource}, {SellSource}", + ProcessorType, CostSource, SellSource); + ; + + return [costDataSource, sellDataSource]; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); + + // L8542-D-DEPARTMENTS + var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542); + + var departmentLookup = new Dictionary(); + if (dictionary?.Records != null) + { + foreach (var dictRecord in dictionary.Records) + { + if (!string.IsNullOrEmpty(dictRecord.Desc1) && !string.IsNullOrEmpty(dictRecord.Code)) + { + departmentLookup[dictRecord.Desc1] = dictRecord.Code; + } + } + + _logger.LogDebug("{ProcessorType}: Loaded {DictCount} department mappings from dictionary", + ProcessorType, departmentLookup.Count); + } + else + { + _logger.LogWarning("{ProcessorType}: Department dictionary (layer 8542) not found or has no records", + ProcessorType); + } + + var firstDataSource = dataSources.First(); + + if (firstDataSource.Records == null || !firstDataSource.Records.Any()) + { + _logger.LogWarning("{ProcessorType}: First data source has no records to process", ProcessorType); + return newRecords; + } + + var groupedData = firstDataSource.Records + .Where(record => record.Code != null && record.Code.Length >= 8 && record.Value1.HasValue) + .Select(record => new + { + Month = record.Code!.Substring(4, 2), + Type = record.Code.Substring(6, 2), + OriginalDepartment = record.Desc1 ?? string.Empty, + Value = record.Value1!.Value, + Department = GetDepartmentByType(record.Code.Substring(6, 2), record.Desc1 ?? string.Empty) + }) + .Select(x => new + { + x.Month, + x.Type, + x.Value, + x.Department, + DepartmentCode = departmentLookup.TryGetValue(x.Department, out var value1) + ? value1 + : x.Department, + FinalCode = + $"2{(departmentLookup.TryGetValue(x.Department, out var value) ? value : x.Department)}{x.Type}{x.Month}" + }) + .GroupBy(x => x.FinalCode) + .Select(group => new + { + FinalCode = group.Key, + Department = group.First().Department, + DepartmentCode = group.First().DepartmentCode, + Type = group.First().Type, + Month = group.First().Month, + TotalValue = group.Sum(x => x.Value) + }) + .ToList(); + + foreach (var groupedRecord in groupedData) + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = groupedRecord.FinalCode, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = groupedRecord.TotalValue, + Desc1 = groupedRecord.Department + }; + + newRecords.Add(newRecord); + + _logger.LogDebug( + "{ProcessorType}: Created summed record {NewRecordCode} with total value {Value} for department {Department} (code: {DepartmentCode}), type {Type}, month {Month}", + ProcessorType, newRecord.Code, newRecord.Value1, groupedRecord.Department, groupedRecord.DepartmentCode, + groupedRecord.Type, groupedRecord.Month); + } + + _logger.LogInformation( + "{ProcessorType}: Processed {GroupCount} unique grouped records from {OriginalCount} original records", + ProcessorType, newRecords.Count, firstDataSource.Records.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + + private string GetDepartmentByType(string type, string originalDepartment) + { + var typesThatUseDepartment = new[] { "02", "09", "10", "11", "12", "13", "14", "15" }; + var typesThatUseAK = new[] { "03", "06", "07", "08" }; + + if (typesThatUseDepartment.Contains(type)) + { + return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment; + } + + if (typesThatUseAK.Contains(type)) + { + return "AK"; + } + + if (type == "04") + { + return "PU"; + } + + if (type == "05") + { + return "OTHER"; + } + + { + _logger.LogWarning("{ProcessorType}: Unknown type {Type}, using {Department}", + nameof(MorskaD6Processor), type, + string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment); + return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment; + } + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 8eb27d1..1dd9b01 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -448,7 +448,8 @@ public class LayersController : Controller "T5-LastValues", "T1-R1", "T4-R2", - "T1-R3" + "T1-R3", + "D6" ]; _logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length); @@ -580,6 +581,16 @@ public class LayersController : Controller processor.Process(processWorker); return; } + case "D6": + { + var processor = _pluginManager.GetProcessor(plugin); + if (processor == null) + { + throw new Exception("D6 processor not found"); + } + processor.Process(processWorker); + return; + } } var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; diff --git a/tools/sql-scripts/admin-yearly/CreateProcessWorker-D6.sql b/tools/sql-scripts/admin-yearly/CreateProcessWorker-D6.sql new file mode 100644 index 0000000..435a5f0 --- /dev/null +++ b/tools/sql-scripts/admin-yearly/CreateProcessWorker-D6.sql @@ -0,0 +1,45 @@ +-- D6 +DECLARE @LayerId UNIQUEIDENTIFIER; +SET @LayerId = NEWID(); + +INSERT INTO [diunabi-morska].[dbo].[Layers] +([Id], [Number], [Name], [CreatedAt], [ModifiedAt], [IsDeleted], + [CreatedById], [ModifiedById], [Type]) +VALUES ((SELECT @LayerId), 8575, 'L8575-A-PW_D6-2025-202506160858', + GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'SourceLayer', 'L4189-P-2025-R3-T1', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'IsEnabled', 'True', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Year', '2025', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Type', 'ProcessWorker', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'ProcessType', 'D6', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Plugin', 'Morska.Process.D6', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Priority', '50', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + diff --git a/tools/sql-scripts/utlis/CreateRecord.sql b/tools/sql-scripts/utlis/CreateRecord.sql index 9c1271b..89f75d5 100644 --- a/tools/sql-scripts/utlis/CreateRecord.sql +++ b/tools/sql-scripts/utlis/CreateRecord.sql @@ -1,5 +1,5 @@ INSERT INTO [diunabi-morska].[dbo].[Records] ([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) -VALUES ((SELECT NEWID()), 'GoogleSheetName-Invoices', 'Raport_R2_Faktury_2024', +VALUES ((SELECT NEWID()), 'SourceLayerSell', 'L4187-P-2025-R1-T1', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, - 'e5315187-5bcb-496a-8c63-4c6575e7e04c'); + 'd1e6e8d2-a13b-421f-84cb-a9ccd5b1c6f5');