using System.Globalization; using System.Text.RegularExpressions; using DiunaBI.Core.Services; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; public class MorskaD6Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.D6"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private string? CostSource { get; set; } private string? SellSource { get; set; } public MorskaD6Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Process(Layer processWorker) { try { _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); // Load configuration from layer records LoadConfiguration(processWorker); // Validate required configuration ValidateConfiguration(); // Perform the actual processing PerformProcessing(processWorker); _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", ProcessorType, processWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); throw; } } private void LoadConfiguration(Layer processWorker) { if (processWorker.Records == null) { throw new InvalidOperationException("ProcessWorker has no records"); } var yearStr = GetRecordValue(processWorker.Records, "Year"); if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { throw new InvalidOperationException("Year record not found or invalid"); } Year = year; CostSource = GetRecordValue(processWorker.Records, "SourceLayerCost"); if (string.IsNullOrEmpty(CostSource)) { throw new InvalidOperationException("SourceLayerCost record not found"); } SellSource = GetRecordValue(processWorker.Records, "SourceLayerSell"); if (string.IsNullOrEmpty(SellSource)) { throw new InvalidOperationException("SourceLayerCosts record not found"); } _logger.LogDebug( "{ProcessorType}: Configuration loaded - Year: {Year}, SourceCost: {CostSource}, SourceSell: {SellSource}", ProcessorType, Year, CostSource, SellSource); } private void ValidateConfiguration() { var errors = new List(); if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); if (string.IsNullOrEmpty(CostSource)) errors.Add("CostSource is required"); if (string.IsNullOrEmpty(SellSource)) errors.Add("SellSource is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); } private void PerformProcessing(Layer processWorker) { _logger.LogDebug( "{ProcessorType}: Processing data for Year: {Year}, CostSource: {CostSource}, SellSource: {SellSource}", ProcessorType, Year, CostSource, SellSource); var processedLayer = GetOrCreateProcessedLayer(processWorker); var dataSources = GetDataSources(); var newRecords = ProcessRecords(dataSources); SaveProcessedLayer(processedLayer, newRecords); _logger.LogInformation( "{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); } private Layer GetOrCreateProcessedLayer(Layer processWorker) { var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (processedLayer == null) { processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, Number = _db.Layers.Count() + 1, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-D6"; _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", ProcessorType, processedLayer.Name); } else { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", ProcessorType, processedLayer.Name); } return processedLayer; } private List GetDataSources() { var costDataSource = _db.Layers .Include(layer => layer.Records!) .AsNoTracking() .FirstOrDefault(x => x.Name == CostSource && !x.IsDeleted && !x.IsCancelled); if (costDataSource == null) { throw new InvalidOperationException($"CostDataSource not found {CostSource}"); } var sellDataSource = _db.Layers .Include(layer => layer.Records!) .AsNoTracking() .FirstOrDefault(x => x.Name == SellSource && !x.IsDeleted && !x.IsCancelled); if (sellDataSource == null) { throw new InvalidOperationException($"SellDataSource not found {SellSource}"); } _logger.LogDebug("{ProcessorType}: Found both data sources data sources: {CostSource}, {SellSource}", ProcessorType, CostSource, SellSource); ; return [costDataSource, sellDataSource]; } private List ProcessRecords(List dataSources) { var newRecords = new List(); // L8542-D-DEPARTMENTS var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542); var departmentLookup = new Dictionary(); if (dictionary?.Records != null) { foreach (var dictRecord in dictionary.Records) { if (!string.IsNullOrEmpty(dictRecord.Desc1) && !string.IsNullOrEmpty(dictRecord.Code)) { departmentLookup[dictRecord.Desc1] = dictRecord.Code; } } _logger.LogDebug("{ProcessorType}: Loaded {DictCount} department mappings from dictionary", ProcessorType, departmentLookup.Count); } else { _logger.LogWarning("{ProcessorType}: Department dictionary (layer 8542) not found or has no records", ProcessorType); } var firstDataSource = dataSources.First(); if (firstDataSource.Records == null || !firstDataSource.Records.Any()) { _logger.LogWarning("{ProcessorType}: First data source has no records to process", ProcessorType); return newRecords; } var groupedData = firstDataSource.Records .Where(record => record.Code != null && record.Code.Length >= 8 && record.Value1.HasValue) .Select(record => new { Month = record.Code!.Substring(4, 2), Type = record.Code.Substring(6, 2), OriginalDepartment = record.Desc1 ?? string.Empty, Value = record.Value1!.Value, Department = GetDepartmentByType(record.Code.Substring(6, 2), record.Desc1 ?? string.Empty) }) .Select(x => new { x.Month, x.Type, x.Value, x.Department, DepartmentCode = departmentLookup.TryGetValue(x.Department, out var value1) ? value1 : x.Department, FinalCode = $"2{(departmentLookup.TryGetValue(x.Department, out var value) ? value : x.Department)}{x.Type}{x.Month}" }) .GroupBy(x => x.FinalCode) .Select(group => new { FinalCode = group.Key, Department = group.First().Department, DepartmentCode = group.First().DepartmentCode, Type = group.First().Type, Month = group.First().Month, TotalValue = group.Sum(x => x.Value) }) .ToList(); foreach (var groupedRecord in groupedData) { var newRecord = new Record { Id = Guid.NewGuid(), Code = groupedRecord.FinalCode, CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = groupedRecord.TotalValue, Desc1 = groupedRecord.Department }; newRecords.Add(newRecord); _logger.LogDebug( "{ProcessorType}: Created summed record {NewRecordCode} with total value {Value} for department {Department} (code: {DepartmentCode}), type {Type}, month {Month}", ProcessorType, newRecord.Code, newRecord.Value1, groupedRecord.Department, groupedRecord.DepartmentCode, groupedRecord.Type, groupedRecord.Month); } _logger.LogInformation( "{ProcessorType}: Processed {GroupCount} unique grouped records from {OriginalCount} original records", ProcessorType, newRecords.Count, firstDataSource.Records.Count); return newRecords; } private void SaveProcessedLayer(Layer processedLayer, List newRecords) { var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); if (!existsInDb) { _db.Layers.Add(processedLayer); _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", ProcessorType, toDelete.Count, layerId); } // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", ProcessorType, records.Count, layerId); } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } private string GetDepartmentByType(string type, string originalDepartment) { var typesThatUseDepartment = new[] { "02", "09", "10", "11", "12", "13", "14", "15" }; var typesThatUseAK = new[] { "03", "06", "07", "08" }; if (typesThatUseDepartment.Contains(type)) { return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment; } if (typesThatUseAK.Contains(type)) { return "AK"; } if (type == "04") { return "PU"; } if (type == "05") { return "OTHER"; } { _logger.LogWarning("{ProcessorType}: Unknown type {Type}, using {Department}", nameof(MorskaD6Processor), type, string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment); return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment; } } }