using System.Globalization; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Services; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; public class MorskaT4R2Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T4.R2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private List? Sources { get; set; } private string? LayerName { get; set; } private string? ReportSheetName { get; set; } private string? InvoicesSheetName { get; set; } public MorskaT4R2Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Process(Layer processWorker) { try { _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); // Load configuration from layer records LoadConfiguration(processWorker); // Validate required configuration ValidateConfiguration(); // Perform the actual processing PerformProcessing(processWorker); _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", ProcessorType, processWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); throw; } } private void LoadConfiguration(Layer processWorker) { if (processWorker.Records == null) { throw new InvalidOperationException("ProcessWorker has no records"); } // Load year var yearStr = GetRecordValue(processWorker.Records, "Year"); if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { throw new InvalidOperationException("Year record not found or invalid"); } Year = year; // Load sources Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); if (Sources.Count == 0) { throw new InvalidOperationException("Source records not found"); } // Load layer name LayerName = GetRecordValue(processWorker.Records, "LayerName"); if (string.IsNullOrEmpty(LayerName)) { throw new InvalidOperationException("LayerName record not found"); } // Load report sheet name ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); if (string.IsNullOrEmpty(ReportSheetName)) { throw new InvalidOperationException("GoogleSheetName record not found"); } // Load invoices sheet name InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices"); if (string.IsNullOrEmpty(InvoicesSheetName)) { throw new InvalidOperationException("GoogleSheetName-Invoices record not found"); } _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}", ProcessorType, Year, Sources.Count, LayerName); } private void ValidateConfiguration() { var errors = new List(); if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required"); if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required"); if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); } private void PerformProcessing(Layer processWorker) { _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", ProcessorType, Year, Sources!.Count); // Get or create processed layer var processedLayer = GetOrCreateProcessedLayer(processWorker); // Process records for all sources var newRecords = ProcessSources(); // Save results SaveProcessedLayer(processedLayer, newRecords); // Update Google Sheets reports UpdateGoogleSheetReport(processedLayer.Id); _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); } private Layer GetOrCreateProcessedLayer(Layer processWorker) { var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (processedLayer == null) { processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, Number = _db.Layers.Count() + 1, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; processedLayer.Name = $"L{processedLayer.Number}-{LayerName}"; _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", ProcessorType, processedLayer.Name); } else { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", ProcessorType, processedLayer.Name); } return processedLayer; } private List ProcessSources() { var newRecords = new List(); foreach (var source in Sources!) { _logger.LogDebug("{ProcessorType}: Processing source {Source}", ProcessorType, source.Desc1); var sourceCodes = GetSourceCodes(source); var sourceRecords = ProcessSourceData(source, sourceCodes); newRecords.AddRange(sourceRecords); _logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records", ProcessorType, source.Desc1, sourceRecords.Count); } _logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}", ProcessorType, newRecords.Count); return newRecords; } private List GetSourceCodes(Record source) { var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}"); if (string.IsNullOrEmpty(rawSourceCodes)) { return new List(); } return ProcessHelper.ParseCodes(rawSourceCodes); } private List ProcessSourceData(Record source, List sourceCodes) { var sourceRecords = new List(); var lastSourceCodes = new List(); // Process monthly data (1-12) for (var month = 1; month <= 12; month++) { var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes); sourceRecords.AddRange(monthRecords); } // Process year summary (month 13) var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes); sourceRecords.AddRange(yearSummaryRecords); return sourceRecords; } private List ProcessMonthData(Record source, List sourceCodes, int month, List lastSourceCodes) { var monthRecords = new List(); if (IsDataAvailableForMonth(month)) { var dataSource = GetMonthDataSource(source, month); if (dataSource != null) { lastSourceCodes.Clear(); lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!)); var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes); var processedRecords = CreateMonthRecords(filteredRecords, source, month); monthRecords.AddRange(processedRecords); _logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records", ProcessorType, month, source.Desc1, processedRecords.Count); } else { _logger.LogWarning("{ProcessorType}: Data source {DataSource} not found", ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T"); } } else { // Future months - create zero value records (except for FK2) if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0) { var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month); monthRecords.AddRange(futureRecords); _logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}", ProcessorType, futureRecords.Count, month); } } return monthRecords; } private bool IsDataAvailableForMonth(int month) { return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year; } private Layer? GetMonthDataSource(Record source, int month) { return _db.Layers .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); } private List FilterRecords(IEnumerable records, List sourceCodes) { return records .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) .ToList(); } private List CreateMonthRecords(List filteredRecords, Record source, int month) { return filteredRecords.Select(x => new Record { Id = Guid.NewGuid(), Code = $"{x.Code}{month:D2}", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, Desc1 = x.Desc1 }).ToList(); } private List CreateFutureMonthRecords(List lastSourceCodes, List sourceCodes, int month) { return lastSourceCodes .Where(x => sourceCodes.Contains(int.Parse(x))) .Select(x => new Record { Id = Guid.NewGuid(), Code = $"{x}{month:D2}", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = 0 }).ToList(); } private List ProcessYearSummaryData(Record source, List sourceCodes) { var dataSourceSum = _db.Layers .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); if (dataSourceSum == null) { _logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found", ProcessorType, $"{Year}/13-{source.Desc1}-T3"); return new List(); } var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes); var yearSummaryRecords = filteredRecords.Select(x => new Record { Id = Guid.NewGuid(), Code = $"{x.Code}13", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = x.Value32 }).ToList(); _logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}", ProcessorType, yearSummaryRecords.Count, source.Desc1); return yearSummaryRecords; } private void SaveProcessedLayer(Layer processedLayer, List newRecords) { var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); if (!existsInDb) { _db.Layers.Add(processedLayer); _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", ProcessorType, toDelete.Count, layerId); } // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", ProcessorType, records.Count, layerId); } private void UpdateGoogleSheetReport(Guid sourceId) { try { _logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}", ProcessorType, sourceId); const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; var processedLayer = GetProcessedLayerData(sourceId); if (processedLayer == null) { throw new InvalidOperationException($"Processed layer {sourceId} not found"); } var codesRow = GetCodesFromSheet(sheetId); UpdateMonthlyData(sheetId, processedLayer, codesRow); UpdateYearSummary(sheetId, processedLayer, codesRow); UpdateTimestamps(sheetId, processedLayer); UpdateInvoicesData(sheetId, processedLayer); _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports", ProcessorType); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}", ProcessorType, sourceId); throw; } } private Layer? GetProcessedLayerData(Guid sourceId) { return _db.Layers .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); } private IList GetCodesFromSheet(string sheetId) { var request = _googleSheetValues.Get(sheetId, "C4:Z4"); var response = request.Execute(); return response.Values[0]; } private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList codesRow) { const int startRow = 6; for (var month = 1; month <= 12; month++) { var values = new List(); var monthStr = month < 10 ? $"0{month}" : month.ToString(); foreach (string code in codesRow) { var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}"); values.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0"); } var valueRange = new ValueRange { Values = new List> { values } }; var row = (startRow + month).ToString(); var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); _logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet", ProcessorType, month); } } private void UpdateYearSummary(string sheetId, Layer processedLayer, IList codesRow) { const int startRow = 6; var valuesSum = new List(); var emptyRow = new List(); foreach (string code in codesRow) { var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); emptyRow.Add(""); valuesSum.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0"); } // Insert empty row before sum var rowEmpty = (startRow + 13).ToString(); var valueRangeEmpty = new ValueRange { Values = new List> { emptyRow } }; var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateEmpty.Execute(); // Update sum row var rowSum = (startRow + 14).ToString(); var valueRangeSum = new ValueRange { Values = new List> { valuesSum } }; var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}"); updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateSum.Execute(); _logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType); } private void UpdateTimestamps(string sheetId, Layer processedLayer) { // Update UTC time var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeUtcTime = new ValueRange { Values = new List> { new List { timeUtc } } }; var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1"); updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeUtc.Execute(); // Update Warsaw time var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeWarsawTime = new ValueRange { Values = new List> { new List { timeWarsaw } } }; var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2"); updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeWarsaw.Execute(); _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", ProcessorType, timeUtc, timeWarsaw); } private void UpdateInvoicesData(string sheetId, Layer processedLayer) { var invoices = processedLayer.Records! .Where(x => x.Code!.Length == 12) .OrderByDescending(x => x.Code) .ToList(); var invoicesValues = new List>(); var cleanUpValues = new List>(); foreach (var invoice in invoices) { var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); var invoiceRow = new List { invoiceDate, "", invoice.Desc1!, invoice.Value1! }; invoicesValues.Add(invoiceRow); var cleanupRow = new List { "", "", "", "" }; cleanUpValues.Add(cleanupRow); } // Clear existing data var cleanupValueRange = new ValueRange { Values = cleanUpValues }; var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; cleanupInvoices.Execute(); // Update with new data var invoicesValueRange = new ValueRange { Values = invoicesValues }; var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateInvoices.Execute(); _logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet", ProcessorType, invoices.Count); } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } }