using System.Globalization; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using DiunaBI.Core.Services.Calculations; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; public class MorskaT1R1Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T1.R1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private List? Sources { get; set; } private List? DynamicCodes { get; set; } private string? GoogleSheetName { get; set; } public MorskaT1R1Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Process(Layer processWorker) { try { _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); // Load configuration from layer records LoadConfiguration(processWorker); // Validate required configuration ValidateConfiguration(); // Perform the actual processing PerformProcessing(processWorker); _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", ProcessorType, processWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); throw; } } private void LoadConfiguration(Layer processWorker) { if (processWorker.Records == null) { throw new InvalidOperationException("ProcessWorker has no records"); } // Load year var yearStr = GetRecordValue(processWorker.Records, "Year"); if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { throw new InvalidOperationException("Year record not found or invalid"); } Year = year; // Load sources Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); if (Sources.Count == 0) { throw new InvalidOperationException("Source records not found"); } // Load dynamic codes DynamicCodes = processWorker.Records .Where(x => x.Code!.Contains("DynamicCode-")) .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) .ToList(); // Load Google Sheet name GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); if (string.IsNullOrEmpty(GoogleSheetName)) { throw new InvalidOperationException("GoogleSheetName record not found"); } _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}", ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName); } private void ValidateConfiguration() { var errors = new List(); if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); } private void PerformProcessing(Layer processWorker) { _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", ProcessorType, Year, Sources!.Count); // Get or create processed layer var processedLayer = GetOrCreateProcessedLayer(processWorker); // Process records for each month var newRecords = ProcessAllMonths(); // Save results SaveProcessedLayer(processedLayer, newRecords); // Update Google Sheet report UpdateGoogleSheetReport(processedLayer.Id); _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); } private Layer GetOrCreateProcessedLayer(Layer processWorker) { var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (processedLayer == null) { processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, Number = _db.Layers.Count() + 1, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1"; _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", ProcessorType, processedLayer.Name); } else { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", ProcessorType, processedLayer.Name); } return processedLayer; } private List ProcessAllMonths() { var newRecords = new List(); for (var month = 1; month < 14; month++) { // Skip future months (except month 13 which is summary) if (Year > DateTime.UtcNow.Year || (Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)) { _logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}", ProcessorType, Year, month); continue; } _logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}", ProcessorType, month, Year); var monthRecords = ProcessSingleMonth(month); newRecords.AddRange(monthRecords); _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}", ProcessorType, monthRecords.Count, month); } return newRecords; } private List ProcessSingleMonth(int month) { var records = new List(); // Collect records from all sources foreach (var source in Sources!) { var sourceRecords = GetSourceRecords(source, month); records.AddRange(sourceRecords); } // Process dynamic codes (calculations) if (DynamicCodes != null && DynamicCodes.Count > 0) { var calculatedRecords = ProcessDynamicCodes(records, month); records.AddRange(calculatedRecords); } // Create final records with month suffix var monthRecords = records.Select(x => new Record { Id = Guid.NewGuid(), Code = $"{x.Code}{month:D2}", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = x.Value32 }).ToList(); return monthRecords; } private List GetSourceRecords(Record source, int month) { var dataSource = _db.Layers .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); if (dataSource == null) { throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found"); } var sourceRecords = new List(); // Check if there are specific codes configured for this source var codesRecord = Sources! .Where(x => x.Code == $"Codes-{source.Desc1}") .FirstOrDefault(); if (codesRecord != null) { var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); sourceRecords.AddRange(dataSource.Records! .Where(x => codes.Contains(int.Parse(x.Code!)))); _logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes", ProcessorType, source.Desc1, codes.Count); } else { sourceRecords.AddRange(dataSource.Records!); _logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records", ProcessorType, source.Desc1, dataSource.Records!.Count); } return sourceRecords; } private List ProcessDynamicCodes(List baseRecords, int month) { var calculatedRecords = new List(); foreach (var dynamicCode in DynamicCodes!) { try { if (string.IsNullOrEmpty(dynamicCode.Desc1)) { _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}", ProcessorType, dynamicCode.Id, month); continue; } var calc = new BaseCalc(dynamicCode.Desc1); if (!calc.IsFormulaCorrect()) { _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}", ProcessorType, calc.Expression, dynamicCode.Id, month); continue; } var calculatedRecord = calc.CalculateT1(baseRecords.Concat(calculatedRecords).ToList()); calculatedRecords.Add(calculatedRecord); _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}", ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32); } catch (Exception e) { _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}", ProcessorType, dynamicCode.Desc1, month); } } return calculatedRecords; } private void SaveProcessedLayer(Layer processedLayer, List newRecords) { var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); if (!existsInDb) { _db.Layers.Add(processedLayer); _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", ProcessorType, toDelete.Count, layerId); } // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", ProcessorType, records.Count, layerId); } private void UpdateGoogleSheetReport(Guid sourceId) { try { _logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}", ProcessorType, GoogleSheetName); const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; // Get processed layer data var processedLayer = _db.Layers .Where(x => x.Id == sourceId) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); if (processedLayer == null) { throw new InvalidOperationException($"Processed layer {sourceId} not found"); } // Get codes from sheet header var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute(); var codesRow = codesResponse.Values[0]; // Update monthly data (months 1-12) UpdateMonthlyData(sheetId, codesRow, processedLayer); // Update summary row (month 13) UpdateSummaryData(sheetId, codesRow, processedLayer); // Update timestamps UpdateTimestamps(sheetId, processedLayer); _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}", ProcessorType, GoogleSheetName); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}", ProcessorType, GoogleSheetName); throw; } } private void UpdateMonthlyData(string sheetId, IList codesRow, Layer processedLayer) { var valueRange = new ValueRange { Values = new List>() }; // Process months 1-12 for (var month = 1; month <= 12; month++) { var monthValues = new List(); foreach (string code in codesRow) { var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}"); monthValues.Add(record?.Value1?.ToString() ?? "0"); } valueRange.Values.Add(monthValues); } var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); _logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType); } private void UpdateSummaryData(string sheetId, IList codesRow, Layer processedLayer) { var valueRange = new ValueRange { Values = new List>() }; // Empty row var emptyRow = new List(); for (int i = 0; i < codesRow.Count; i++) { emptyRow.Add(""); } valueRange.Values.Add(emptyRow); // Summary row (month 13) var summaryValues = new List(); foreach (string code in codesRow) { var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); summaryValues.Add(record?.Value1?.ToString() ?? "0"); } valueRange.Values.Add(summaryValues); var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); _logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType); } private void UpdateTimestamps(string sheetId, Layer processedLayer) { var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeTime = new ValueRange { Values = new List> { new List { timeUtc }, new List { timeWarsaw } } }; var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2"); updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTime.Execute(); _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", ProcessorType, timeUtc, timeWarsaw); } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } }