using System.Text; using System.Text.Json; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Plugins; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.PedrolloPL.Importers; public class PedrolloPLImportB3 : BaseDataImporter { public override string ImporterType => "PedrolloPL.Import.B3"; private readonly AppDbContext _db; private readonly ILogger _logger; // Configuration properties private string? DataInboxName { get; set; } private string? DataInboxSource { get; set; } private string? StartDate { get; set; } private string? EndDate { get; set; } private string? ImportYear { get; set; } private bool IsEnabled { get; set; } // Cached deserialized data private List>? _cachedRawData; private DataInbox? _cachedDataInbox; private Dictionary? _regionCodeMap; public PedrolloPLImportB3( AppDbContext db, ILogger logger) { _db = db; _logger = logger; } public override void Import(Layer importWorker) { try { _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); // Clear cache at start _cachedRawData = null; _cachedDataInbox = null; _regionCodeMap = null; LoadConfiguration(importWorker); ValidateConfiguration(); if (!IsEnabled) { _logger.LogInformation("{ImporterType}: Import disabled for {ImportWorkerName}", ImporterType, importWorker.Name); return; } // Find and deserialize DataInbox data FindAndDeserializeDataInbox(); // Load region code mapping from dictionary layer LoadRegionCodeMapping(); // Map data from DataInbox to Layer records var mappedRecords = MapDataToRecords(); // Create new Import layer var importLayer = CreateImportLayer(importWorker); // Save records to database SaveRecordsToLayer(importLayer, mappedRecords); _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName} - Created {RecordCount} records", ImporterType, importWorker.Name, mappedRecords.Count); } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); throw; } finally { // Clear cache after import _cachedRawData = null; _cachedDataInbox = null; _regionCodeMap = null; } } private void LoadConfiguration(Layer importWorker) { if (importWorker.Records == null) return; DataInboxName = GetRecordValue(importWorker.Records, "DataInboxName"); DataInboxSource = GetRecordValue(importWorker.Records, "DataInboxSource"); StartDate = GetRecordValue(importWorker.Records, "StartDate"); EndDate = GetRecordValue(importWorker.Records, "EndDate"); ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; _logger.LogDebug( "{ImporterType}: Configuration loaded - DataInboxName: {DataInboxName}, Source: {Source}, Year: {Year}, Period: {StartDate} to {EndDate}, Enabled: {IsEnabled}", ImporterType, DataInboxName, DataInboxSource, ImportYear, StartDate, EndDate, IsEnabled); } private void ValidateConfiguration() { var errors = new List(); if (string.IsNullOrEmpty(DataInboxName)) errors.Add("DataInboxName is required"); if (string.IsNullOrEmpty(DataInboxSource)) errors.Add("DataInboxSource is required"); if (string.IsNullOrEmpty(StartDate)) errors.Add("StartDate is required"); if (string.IsNullOrEmpty(EndDate)) errors.Add("EndDate is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } _logger.LogDebug("{ImporterType}: Configuration validated successfully", ImporterType); } private void FindAndDeserializeDataInbox() { _logger.LogDebug("{ImporterType}: Searching for DataInbox with Name='{DataInboxName}' and Source='{DataInboxSource}'", ImporterType, DataInboxName, DataInboxSource); // Find DataInbox by Name and Source, order by CreatedAt descending to get the latest var dataInbox = _db.DataInbox .Where(x => x.Name == DataInboxName && x.Source == DataInboxSource) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (dataInbox == null) { throw new InvalidOperationException( $"DataInbox not found with Name='{DataInboxName}' and Source='{DataInboxSource}'"); } _logger.LogInformation("{ImporterType}: Found DataInbox - Id: {DataInboxId}, Name: {Name}, Source: {Source}, CreatedAt: {CreatedAt}", ImporterType, dataInbox.Id, dataInbox.Name, dataInbox.Source, dataInbox.CreatedAt); // Deserialize the data try { var data = Convert.FromBase64String(dataInbox.Data); var jsonString = Encoding.UTF8.GetString(data); _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64", ImporterType, data.Length); // Deserialize as array of arrays: [["", 1183.15, ...], ["DOLNOŚLĄSKIE", ...]] var rawData = JsonSerializer.Deserialize>>(jsonString); if (rawData == null || rawData.Count == 0) { throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}"); } _logger.LogInformation("{ImporterType}: Successfully deserialized {RowCount} rows from DataInbox", ImporterType, rawData.Count); // Log first few rows for debugging if (rawData.Count > 0) { var sampleSize = Math.Min(3, rawData.Count); _logger.LogDebug("{ImporterType}: Sample rows (first {SampleSize}):", ImporterType, sampleSize); for (int i = 0; i < sampleSize; i++) { var row = rawData[i]; if (row.Count > 0) { var regionName = row[0]?.ToString() ?? "null"; var valueCount = row.Count - 1; _logger.LogDebug(" [{Index}] Region: {Region}, Values: {ValueCount}", i, regionName, valueCount); } } } // Cache the deserialized data _cachedRawData = rawData; _cachedDataInbox = dataInbox; } catch (FormatException e) { _logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxId}", ImporterType, dataInbox.Id); throw new InvalidOperationException($"Invalid base64 data in DataInbox: {dataInbox.Name}", e); } catch (JsonException e) { _logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxId}", ImporterType, dataInbox.Id); throw new InvalidOperationException($"Invalid JSON data in DataInbox: {dataInbox.Name}", e); } } private void LoadRegionCodeMapping() { const string dictionaryLayerName = "L1-D-P2-CODES"; _logger.LogDebug("{ImporterType}: Loading region code mapping from dictionary layer '{DictionaryLayerName}'", ImporterType, dictionaryLayerName); var dictionaryLayer = _db.Layers .Where(x => x.Name == dictionaryLayerName && x.Type == LayerType.Dictionary) .FirstOrDefault(); if (dictionaryLayer == null) { throw new InvalidOperationException($"Dictionary layer '{dictionaryLayerName}' not found"); } // Load records for the dictionary layer var records = _db.Records .Where(x => x.LayerId == dictionaryLayer.Id) .ToList(); // Build mapping: Desc1 (region name) -> Code _regionCodeMap = records.ToDictionary( r => r.Desc1 ?? string.Empty, r => r.Code ?? string.Empty, StringComparer.OrdinalIgnoreCase); _logger.LogInformation("{ImporterType}: Loaded {MappingCount} region code mappings", ImporterType, _regionCodeMap.Count); } private List MapDataToRecords() { if (_cachedRawData == null) { throw new InvalidOperationException("Raw data not loaded. Call FindAndDeserializeDataInbox first."); } if (_regionCodeMap == null) { throw new InvalidOperationException("Region code mapping not loaded. Call LoadRegionCodeMapping first."); } var records = new List(); var now = DateTime.UtcNow; _logger.LogDebug("{ImporterType}: Starting data mapping for {RowCount} rows", ImporterType, _cachedRawData.Count); foreach (var row in _cachedRawData) { if (row.Count < 13) { _logger.LogWarning("{ImporterType}: Skipping row with insufficient data - expected 13 elements, got {Count}", ImporterType, row.Count); continue; } // First element is region name var regionName = row[0]?.ToString(); if (string.IsNullOrEmpty(regionName)) { _logger.LogWarning("{ImporterType}: Skipping row with empty region name", ImporterType); continue; } // Find region code from dictionary if (!_regionCodeMap.TryGetValue(regionName, out var regionCode)) { _logger.LogWarning("{ImporterType}: Region code not found for '{RegionName}' - skipping", ImporterType, regionName); continue; } // Create 12 records (one per month) for (int month = 1; month <= 12; month++) { var valueIndex = month; // row[1] = January, row[2] = February, etc. var valueObj = row[valueIndex]; // Convert value to double double? value = null; if (valueObj != null) { // Handle JsonElement if deserialized from JSON if (valueObj is JsonElement jsonElement) { if (jsonElement.ValueKind == JsonValueKind.Number) { value = jsonElement.GetDouble(); } } else if (valueObj is double d) { value = d; } else if (double.TryParse(valueObj.ToString(), out var parsed)) { value = parsed; } } // Create code: {regionCode}{month:00} var code = $"{regionCode}{month:00}"; var record = new Record { Id = Guid.NewGuid(), Code = code, Value1 = value, CreatedAt = now, ModifiedAt = now }; records.Add(record); } _logger.LogDebug("{ImporterType}: Mapped region '{RegionName}' (code: {RegionCode}) to 12 records", ImporterType, regionName, regionCode); } _logger.LogInformation("{ImporterType}: Successfully mapped {RecordCount} records from {RowCount} rows", ImporterType, records.Count, _cachedRawData.Count); return records; } private Layer CreateImportLayer(Layer importWorker) { var now = DateTime.UtcNow; var importLayer = new Layer { Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, IsCancelled = false, CreatedAt = now, ModifiedAt = now, CreatedById = Guid.Parse("f392209e-123e-4651-a5a4-0b1d6cf9ff9d"), // System user ModifiedById = Guid.Parse("f392209e-123e-4651-a5a4-0b1d6cf9ff9d") // System user }; importLayer.Name = $"L{importLayer.Number}-I-B3-{ImportYear}-{now:yyyyMMddHHmm}"; _logger.LogDebug("{ImporterType}: Creating import layer '{LayerName}' (Number: {Number})", ImporterType, importLayer.Name, importLayer.Number); _db.Layers.Add(importLayer); _db.SaveChanges(); _logger.LogInformation("{ImporterType}: Created import layer '{LayerName}' with Id: {LayerId}", ImporterType, importLayer.Name, importLayer.Id); return importLayer; } private void SaveRecordsToLayer(Layer importLayer, List records) { _logger.LogDebug("{ImporterType}: Saving {RecordCount} records to layer {LayerId}", ImporterType, records.Count, importLayer.Id); // Delete any existing records for this layer (shouldn't be any, but just in case) var toDelete = _db.Records.Where(x => x.LayerId == importLayer.Id).ToList(); if (toDelete.Count > 0) { _logger.LogWarning("{ImporterType}: Found {ExistingCount} existing records for layer {LayerId}, removing them", ImporterType, toDelete.Count, importLayer.Id); _db.Records.RemoveRange(toDelete); } // Set all required properties for each record foreach (var record in records) { record.LayerId = importLayer.Id; record.CreatedById = Guid.Parse("f392209e-123e-4651-a5a4-0b1d6cf9ff9d"); // System user record.ModifiedById = Guid.Parse("f392209e-123e-4651-a5a4-0b1d6cf9ff9d"); // System user _db.Records.Add(record); } _db.SaveChanges(); _logger.LogInformation("{ImporterType}: Successfully saved {RecordCount} records to layer '{LayerName}'", ImporterType, records.Count, importLayer.Name); } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } }