using System.Globalization; using System.Text.RegularExpressions; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Plugins; using DiunaBI.Infrastructure.Services; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; public class MorskaT1R3Processor : BaseDataProcessor { public override string ProcessorType => "Morska.Process.T1.R3"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private string? Source { get; set; } public MorskaT1R3Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Process(Layer processWorker) { try { _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); // Load configuration from layer records LoadConfiguration(processWorker); // Validate required configuration ValidateConfiguration(); // Perform the actual processing PerformProcessing(processWorker); _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", ProcessorType, processWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", ProcessorType, processWorker.Name, processWorker.Id); throw; } } private void LoadConfiguration(Layer processWorker) { if (processWorker.Records == null) { throw new InvalidOperationException("ProcessWorker has no records"); } // Load year var yearStr = GetRecordValue(processWorker.Records, "Year"); if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { throw new InvalidOperationException("Year record not found or invalid"); } Year = year; // Load source Source = GetRecordValue(processWorker.Records, "Source"); if (string.IsNullOrEmpty(Source)) { throw new InvalidOperationException("Source record not found"); } _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", ProcessorType, Year, Source); } private void ValidateConfiguration() { var errors = new List(); if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); } private void PerformProcessing(Layer processWorker) { _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}", ProcessorType, Year, Source); // Get or create processed layer var processedLayer = GetOrCreateProcessedLayer(processWorker); // Get data sources var dataSources = GetDataSources(); // Process records var newRecords = ProcessRecords(dataSources); // Save results SaveProcessedLayer(processedLayer, newRecords); // Update Google Sheet report UpdateGoogleSheetReport(processedLayer.Id); _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); } private Layer GetOrCreateProcessedLayer(Layer processWorker) { var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (processedLayer == null) { processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, Number = _db.Layers.Count() + 1, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1"; _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", ProcessorType, processedLayer.Name); } else { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", ProcessorType, processedLayer.Name); } return processedLayer; } private List GetDataSources() { string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$"; var dataSources = _db.Layers .Where(x => !x.IsDeleted && !x.IsCancelled) .Include(layer => layer.Records!) .AsNoTracking() .AsEnumerable() .Where(x => Regex.IsMatch(x.Name!, pattern)) .ToList(); if (dataSources.Count == 0) { throw new InvalidOperationException($"No data sources found for pattern: {pattern}"); } _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}", ProcessorType, dataSources.Count, pattern); return dataSources; } private List ProcessRecords(List dataSources) { var newRecords = new List(); foreach (var dataSource in dataSources) { var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); if (monthStr == null || !int.TryParse(monthStr, out var month)) { _logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}", ProcessorType, dataSource.Name); continue; } _logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}", ProcessorType, dataSource.Name, month); var sourceRecords = ProcessDataSourceRecords(dataSource, month); newRecords.AddRange(sourceRecords); _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}", ProcessorType, sourceRecords.Count, dataSource.Name); } _logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}", ProcessorType, newRecords.Count); return newRecords; } private List ProcessDataSourceRecords(Layer dataSource, int month) { var newRecords = new List(); // L8542-D-DEPARTMENTS var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542); foreach (var record in dataSource.Records!) { if (record.Value1 == null) { _logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null", ProcessorType, record.Code); continue; } // Process values for positions 1-32 for (var i = 1; i < 33; i++) { var value = ProcessHelper.GetValue(record, i); if (value == null) { continue; } var baseValue = (double)record.Value1!; var positionValue = (double)value; var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100; var newRecord = new Record { Id = Guid.NewGuid(), Code = $"{record.Code}{month:D2}{i:D2}", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, Value1 = calculatedValue, Desc1 = record.Desc1 }; newRecords.Add(newRecord); _logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}", ProcessorType, newRecord.Code, newRecord.Value1, record.Code); } } return newRecords; } private void SaveProcessedLayer(Layer processedLayer, List newRecords) { var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); if (!existsInDb) { _db.Layers.Add(processedLayer); _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", ProcessorType, toDelete.Count, layerId); } // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", ProcessorType, records.Count, layerId); } private void UpdateGoogleSheetReport(Guid sourceId) { try { _logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}", ProcessorType, sourceId); const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; var processedLayer = _db.Layers .Where(x => x.Id == sourceId) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); if (processedLayer == null) { throw new InvalidOperationException($"Processed layer {sourceId} not found"); } // Update sheets for all months for (var month = 1; month <= 12; month++) { UpdateMonthSheet(sheetId, processedLayer, month); Thread.Sleep(1000); } _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months", ProcessorType); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}", ProcessorType, sourceId); throw; } } private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month) { var sheetName = ProcessHelper.GetSheetName(month, Year); try { _logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}", ProcessorType, sheetName, month); // Get codes from sheet ValueRange? dataRangeResponse; try { dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); } catch (Exception e) { _logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}", ProcessorType, sheetName, e.Message); return; } if (dataRangeResponse?.Values == null) { _logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping", ProcessorType, sheetName); return; } // Update data UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month); // Update timestamps UpdateSheetTimestamps(sheetId, sheetName, processedLayer); _logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}", ProcessorType, sheetName); } catch (Exception e) { _logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}", ProcessorType, sheetName, month); throw; } } private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList> codeRows, int month) { var updateValueRange = new ValueRange { Values = new List>() }; foreach (var row in codeRows) { if (row.Count == 0) continue; var code = row[0].ToString(); var updateRow = new List(); var department = ""; // Process columns C to Q (positions 1-15) for (var position = 1; position <= 15; position++) { var recordCode = $"{code}{month:D2}{position:D2}"; var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode); if (codeRecord?.Value1 != null) { updateRow.Add(codeRecord.Value1); _logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}", ProcessorType, codeRecord.Value1, recordCode); } else { updateRow.Add(""); } department = codeRecord?.Desc1 ?? ""; } updateRow.Add(department); updateValueRange.Values.Add(updateRow); } // Update sheet with new values var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:R200"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); _logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}", ProcessorType, updateValueRange.Values.Count, sheetName); } private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer) { var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeTime = new ValueRange { Values = new List> { new List { timeUtc }, new List { timeWarsaw } } }; var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTime.Execute(); _logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", ProcessorType, sheetName, timeUtc, timeWarsaw); } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } }