after refactor cleanup

This commit is contained in:
2025-11-28 11:21:22 +01:00
parent 5db6de1503
commit 07423023a0
305 changed files with 80 additions and 13326 deletions

View File

@@ -0,0 +1,13 @@
using DiunaBI.Domain.Entities;
using DiunaBI.Infrastructure.Interfaces;
namespace DiunaBI.Plugins.Morska.Importers;
public abstract class MorskaBaseImporter : IDataImporter
{
public abstract string ImporterType { get; }
public virtual bool CanImport(string importerType) => ImporterType == importerType;
public abstract void Import(Layer importWorker);
}

View File

@@ -0,0 +1,415 @@
using System.Globalization;
using DiunaBI.Domain.Entities;
using DiunaBI.Infrastructure.Data;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaD1Importer : MorskaBaseImporter
{
public override string ImporterType => "Morska.Import.D1";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD1Importer> _logger;
// Configuration properties
private string? SheetId { get; set; }
private string? SheetTabName { get; set; }
private string? DataRange { get; set; }
private string? ImportYear { get; set; }
private string? ImportMonth { get; set; }
private string? ImportName { get; set; }
private DateTime? StartDate { get; set; }
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaD1Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaD1Importer> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
try
{
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker))
{
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return;
}
ValidateConfiguration();
PerformImport(importWorker);
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
_cachedSheetData = null;
_cachedDataKey = null;
}
}
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
{
if (importWorker.Records == null) return;
SheetId = GetRecordValue(importWorker.Records, "SheetId");
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
DataRange = GetRecordValue(importWorker.Records, "DataRange");
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
ImportName = GetRecordValue(importWorker.Records, "ImportName");
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
{
StartDate = startDate;
}
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
{
EndDate = endDate;
}
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
private bool ShouldPerformImport(Layer importWorker)
{
if (!IsEnabled)
{
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
if (StartDate.HasValue && EndDate.HasValue)
{
var now = DateTime.UtcNow.Date;
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
{
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
if (!IsImportedLayerUpToDate(importWorker))
{
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
return true;
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
}
private bool IsImportedLayerUpToDate(Layer importWorker)
{
var newestLayer = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == importWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.FirstOrDefault();
if (newestLayer == null)
{
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, need to create new layer",
ImporterType, importWorker.Name);
return false;
}
try
{
var data = GetSheetData();
if (data == null || data.Count < 2)
{
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
var isUpToDate = true;
foreach (var row in data)
{
if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
var code = row[0].ToString();
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
if (record == null)
{
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
ImporterType, code, importWorker.Name);
isUpToDate = false;
continue;
}
// Check values 3-17 (Value1-Value15)
for (int i = 3; i <= 17 && i < row.Count; i++)
{
var sheetValue = ParseValue(row[i]?.ToString());
var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
{
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}",
ImporterType, code, i - 2, dbValue, sheetValue);
isUpToDate = false;
}
}
// check also Desc1 from records (Value18 in GSheet)
if (row.Count <= 18) continue;
var sheetDesc1 = row[18]?.ToString();
var recordDesc1 = record.Desc1;
var normalizedSheetDesc1 = string.IsNullOrEmpty(sheetDesc1) ? null : sheetDesc1;
var normalizedRecordDesc1 = string.IsNullOrEmpty(recordDesc1) ? null : recordDesc1;
if (normalizedSheetDesc1 == normalizedRecordDesc1) continue;
_logger.LogDebug("{ImporterType}: Desc1 mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
ImporterType, code, normalizedRecordDesc1, normalizedSheetDesc1);
isUpToDate = false;
}
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
return isUpToDate;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
ImporterType, importWorker.Name);
throw;
}
}
private void PerformImport(Layer importWorker)
{
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
ImporterType, SheetId, SheetTabName, DataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
try
{
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
var newRecords = (from t in data
where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
select new Record
{
Id = Guid.NewGuid(),
Code = t[0].ToString(),
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
Desc1 = IndexExists(t, 18) ? t[18]?.ToString() : null,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}).ToList();
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
ImporterType, newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}
private double? ParseValue(string? value)
{
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
try
{
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result;
}
catch (FormatException e)
{
_logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}",
ImporterType, value, e.Message);
return null;
}
}
private bool IndexExists(IList<object> array, int index)
{
return array != null && index >= 0 && index < array.Count;
}
private double? GetRecordValueByIndex(Record record, int valueIndex)
{
return valueIndex switch
{
1 => record.Value1,
2 => record.Value2,
3 => record.Value3,
4 => record.Value4,
5 => record.Value5,
6 => record.Value6,
7 => record.Value7,
8 => record.Value8,
9 => record.Value9,
10 => record.Value10,
11 => record.Value11,
12 => record.Value12,
13 => record.Value13,
14 => record.Value14,
15 => record.Value15,
_ => null
};
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
ImporterType, records.Count, layerId);
}
}

View File

@@ -0,0 +1,428 @@
using System.Globalization;
using System.Text;
using System.Text.Json;
using DiunaBI.Domain.Entities;
using DiunaBI.Infrastructure.Data;
using DiunaBI.Infrastructure.Services;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaD3Importer : MorskaBaseImporter
{
public override string ImporterType => "Morska.Import.D3";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD3Importer> _logger;
// Configuration properties
private string? ImportYear { get; set; }
private string? ImportMonth { get; set; }
private string? ImportName { get; set; }
private string? ImportType { get; set; }
private string? SheetId { get; set; }
private bool IsEnabled { get; set; }
// Cached deserialized data
private List<Record>? _cachedRecords;
private string? _cachedDataInboxId;
public MorskaD3Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaD3Importer> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
try
{
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// Clear cache at start
_cachedRecords = null;
_cachedDataInboxId = null;
LoadConfiguration(importWorker);
// Deserialize data early - right after LoadConfiguration
DeserializeDataInboxData();
if (!ShouldPerformImport(importWorker))
{
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return;
}
ValidateConfiguration();
PerformImport(importWorker);
// Export to Google Sheets after successful import
ExportToGoogleSheets();
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// Clear cache after import
_cachedRecords = null;
_cachedDataInboxId = null;
}
}
private void LoadConfiguration(Layer importWorker)
{
if (importWorker.Records == null) return;
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
ImportName = GetRecordValue(importWorker.Records, "ImportName");
ImportType = GetRecordValue(importWorker.Records, "ImportType");
SheetId = GetRecordValue(importWorker.Records, "SheetId");
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
_logger.LogDebug(
"{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}, SheetId: {SheetId}",
ImporterType, importWorker.Name, ImportType, SheetId);
}
private void DeserializeDataInboxData()
{
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
if (dataInbox == null)
{
throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
}
_logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
ImporterType, dataInbox.Id, dataInbox.CreatedAt);
try
{
var data = Convert.FromBase64String(dataInbox.Data);
var jsonString = Encoding.UTF8.GetString(data);
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
ImporterType, data.Length);
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
if (records == null)
{
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
}
_logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
ImporterType, records.Count);
// Cache the deserialized data
_cachedRecords = records;
_cachedDataInboxId = dataInbox.Id.ToString();
}
catch (FormatException e)
{
_logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxName}",
ImporterType, ImportType);
throw new InvalidOperationException($"Invalid base64 data in DataInbox: {ImportType}", e);
}
catch (JsonException e)
{
_logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxName}",
ImporterType, ImportType);
throw new InvalidOperationException($"Invalid JSON data in DataInbox: {ImportType}", e);
}
}
private List<Record> GetFilteredRecords()
{
if (_cachedRecords == null)
{
throw new InvalidOperationException("Data has not been deserialized yet");
}
var filteredRecords = _cachedRecords.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
if (filteredRecords.Count == 0)
{
throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
}
_logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
ImporterType, filteredRecords.Count, ImportYear, ImportMonth);
return filteredRecords;
}
private bool ShouldPerformImport(Layer importWorker)
{
if (!IsEnabled)
{
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
if (dataInbox == null)
{
_logger.LogDebug("{ImporterType}: No DataInbox found for type {ImportType}",
ImporterType, ImportType);
return false;
}
// Check if imported layer is up to date
if (!IsImportedLayerUpToDate(importWorker))
{
_logger.LogDebug("{ImporterType}: Layer is out of date, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
_logger.LogDebug("{ImporterType}: Layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
}
private bool IsImportedLayerUpToDate(Layer importWorker)
{
var newestLayer = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.ParentId == importWorker.Id
&& x.Name!.Contains($"I-{ImportName}-{ImportYear}/{ImportMonth}"))
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.FirstOrDefault();
if (newestLayer == null)
{
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, import needed",
ImporterType, importWorker.Name);
return false;
}
try
{
var currentRecords = GetFilteredRecords();
// Compare record counts first
if (newestLayer.Records?.Count != currentRecords.Count)
{
_logger.LogDebug("{ImporterType}: Record count mismatch - DB: {DbCount}, DataInbox: {DataCount}",
ImporterType, newestLayer.Records?.Count ?? 0, currentRecords.Count);
return false;
}
// Compare individual records
foreach (var currentRecord in currentRecords)
{
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == currentRecord.Code);
if (existingRecord == null)
{
_logger.LogDebug("{ImporterType}: Record with code {Code} not found in existing layer",
ImporterType, currentRecord.Code);
return false;
}
// Compare all relevant fields
if (Math.Abs((existingRecord.Value1 ?? 0) - (currentRecord.Value1 ?? 0)) > 0.001 ||
existingRecord.Desc1 != currentRecord.Desc1)
{
_logger.LogDebug("{ImporterType}: Data difference found for code {Code}",
ImporterType, currentRecord.Code);
return false;
}
}
_logger.LogDebug("{ImporterType}: All records match, layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
ImporterType, importWorker.Name);
return false;
}
}
private void PerformImport(Layer importWorker)
{
_logger.LogDebug("{ImporterType}: Starting import for DataInbox type {ImportType}",
ImporterType, ImportType);
var filteredRecords = GetFilteredRecords();
// Prepare records for saving
var recordsToSave = filteredRecords.Select(x =>
{
x.Id = Guid.NewGuid();
x.CreatedAt = DateTime.UtcNow;
x.ModifiedAt = DateTime.UtcNow;
return x;
}).ToList();
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
_db.Layers.Add(layer);
SaveRecords(layer.Id, recordsToSave);
_db.SaveChanges();
_logger.LogInformation(
"{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
ImporterType, recordsToSave.Count, layer.Name, layer.Id);
}
private void ExportToGoogleSheets()
{
if (string.IsNullOrEmpty(SheetId))
{
_logger.LogWarning("{ImporterType}: SheetId not configured, skipping Google Sheets export",
ImporterType);
return;
}
try
{
_logger.LogDebug("{ImporterType}: Starting Google Sheets export to {SheetId}",
ImporterType, SheetId);
var filteredRecords = GetFilteredRecords();
var sheetTabName = ProcessHelper.GetSheetName(int.Parse(ImportMonth!), int.Parse(ImportYear!));
_logger.LogDebug("{ImporterType}: Using sheet tab name: {SheetTabName}",
ImporterType, sheetTabName);
// Get current sheet data
var currentSheetData = _googleSheetValues.Get(SheetId!, $"{sheetTabName}!C7:D200").Execute();
var updateRequests = new List<ValueRange>();
for (var rowIndex = 0; rowIndex < 194; rowIndex++)
{
if (currentSheetData.Values == null || rowIndex >= currentSheetData.Values.Count)
continue;
var existingRow = currentSheetData.Values[rowIndex];
if (existingRow.Count == 0 || string.IsNullOrEmpty(existingRow[0]?.ToString()))
continue;
var accountCode = existingRow[0].ToString()!;
var matchingRecord = filteredRecords.FirstOrDefault(x => x.Desc1 == accountCode);
if (matchingRecord == null)
continue;
var newValue = matchingRecord.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0";
var existingValue = (existingRow.Count > 1 ? existingRow[1] : "")?.ToString();
if (existingValue != newValue)
{
var range = $"{sheetTabName}!D{rowIndex + 7}";
updateRequests.Add(new ValueRange
{
Range = range,
Values = new List<IList<object>> { new List<object> { newValue } }
});
_logger.LogDebug("{ImporterType}: Scheduled update for {AccountCode} at {Range} with value {Value}",
ImporterType, accountCode, range, newValue);
}
}
if (updateRequests.Count > 0)
{
var batchUpdate = new BatchUpdateValuesRequest
{
Data = updateRequests,
ValueInputOption = "USER_ENTERED"
};
_googleSheetValues.BatchUpdate(batchUpdate, SheetId!).Execute();
_logger.LogInformation("{ImporterType}: Updated {Count} cells in Google Sheet {SheetId}",
ImporterType, updateRequests.Count, SheetId);
}
else
{
_logger.LogInformation("{ImporterType}: No changes to export to Google Sheet {SheetId}",
ImporterType, SheetId);
}
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to export to Google Sheets {SheetId}",
ImporterType, SheetId);
throw;
}
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
ImporterType, records.Count, layerId);
}
}

View File

@@ -0,0 +1,397 @@
using System.Globalization;
using DiunaBI.Domain.Entities;
using DiunaBI.Infrastructure.Data;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaFk2Importer : MorskaBaseImporter
{
public override string ImporterType => "Morska.Import.FK2";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaFk2Importer> _logger;
// Configuration properties
private string? SheetId { get; set; }
private string? SheetTabName { get; set; }
private string? DataRange { get; set; }
private string? ImportYear { get; set; }
private string? ImportMonth { get; set; }
private string? ImportName { get; set; }
private DateTime? StartDate { get; set; }
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaFk2Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaFk2Importer> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
try
{
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// ✅ Clear cache at start
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker))
{
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return;
}
ValidateConfiguration();
PerformImport(importWorker);
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// ✅ Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
// ✅ Dodaj metodę cache
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
{
if (importWorker.Records == null) return;
SheetId = GetRecordValue(importWorker.Records, "SheetId");
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
DataRange = GetRecordValue(importWorker.Records, "DataRange");
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
ImportName = GetRecordValue(importWorker.Records, "ImportName");
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
{
StartDate = startDate;
}
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
{
EndDate = endDate;
}
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
private bool ShouldPerformImport(Layer importWorker)
{
if (!IsEnabled)
{
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
if (StartDate.HasValue && EndDate.HasValue)
{
var now = DateTime.UtcNow.Date;
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
{
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
if (!IsImportedLayerUpToDate(importWorker))
{
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
return true;
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
}
private bool IsImportedLayerUpToDate(Layer importWorker)
{
var newestLayer = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == importWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.FirstOrDefault();
if (newestLayer == null)
{
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
ImporterType, importWorker.Name);
return true;
}
try
{
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
if (data == null || data.Count == 0)
{
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
var isUpToDate = true;
for (var i = 0; i < data.Count; i++)
{
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue;
try
{
var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3) continue;
var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
if (record == null)
{
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
ImporterType, code, importWorker.Name);
isUpToDate = false;
continue;
}
if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) &&
Math.Abs((double)(record.Value1 - value)!) >= 0.01)
{
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
ImporterType, code, record.Value1, value);
isUpToDate = false;
}
if (record.Desc1 != data[i][3]?.ToString())
{
_logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}",
ImporterType, code, record.Desc1, data[i][3]?.ToString());
isUpToDate = false;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison",
ImporterType, i);
isUpToDate = false;
}
}
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
return isUpToDate;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
ImporterType, importWorker.Name);
throw;
}
}
private void PerformImport(Layer importWorker)
{
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
ImporterType, SheetId, SheetTabName, DataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>();
try
{
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
if (data != null)
{
for (var i = 0; i < data.Count; i++)
{
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
{
_logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc",
ImporterType, i);
continue;
}
var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3)
{
_logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}",
ImporterType, i, data[i][1]);
throw new InvalidOperationException($"Invalid date in row {i}");
}
var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value",
ImporterType, i);
continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][3].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
}
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
ImporterType, newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
ImporterType, records.Count, layerId);
}
}

View File

@@ -0,0 +1,360 @@
using System.Globalization;
using DiunaBI.Domain.Entities;
using DiunaBI.Infrastructure.Data;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaStandardImporter : MorskaBaseImporter
{
public override string ImporterType => "Morska.Import.Standard";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaStandardImporter> _logger;
// Configuration properties
private string? SheetId { get; set; }
private string? SheetTabName { get; set; }
private string? DataRange { get; set; }
private string? ImportYear { get; set; }
private string? ImportMonth { get; set; }
private string? ImportName { get; set; }
private DateTime? StartDate { get; set; }
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaStandardImporter(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaStandardImporter> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Import(Layer importWorker)
{
try
{
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// Clear cache before import
_cachedSheetData = null;
_cachedDataKey = null;
// Load configuration from layer records
LoadConfiguration(importWorker);
// Check if import should be performed
if (!ShouldPerformImport(importWorker))
{
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return;
}
// Validate required configuration
ValidateConfiguration();
// Perform the actual import
PerformImport(importWorker);
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
{
if (importWorker.Records == null) return;
SheetId = GetRecordValue(importWorker.Records, "SheetId");
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
DataRange = GetRecordValue(importWorker.Records, "DataRange");
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
ImportName = GetRecordValue(importWorker.Records, "ImportName");
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
{
StartDate = startDate;
}
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
{
EndDate = endDate;
}
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
private bool ShouldPerformImport(Layer importWorker)
{
if (!IsEnabled)
{
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
// Check date range if configured
if (StartDate.HasValue && EndDate.HasValue)
{
var now = DateTime.UtcNow.Date;
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
{
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
// Outside date range - check if imported layer is up to date
if (!IsImportedLayerUpToDate(importWorker))
{
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
// No date constraints - always import
return true;
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
}
private bool IsImportedLayerUpToDate(Layer importWorker)
{
var newestLayer = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == importWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.FirstOrDefault();
if (newestLayer == null)
{
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
ImporterType, importWorker.Name);
return true;
}
try
{
var data = GetSheetData();
if (data == null || data.Count < 2)
{
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
// Check if the number of columns matches
if (data[0].Count != data[1].Count)
{
_logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
for (var i = 0; i < data[1].Count; i++)
{
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
ImporterType, i);
continue;
}
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (existingRecord == null || existingRecord.Value1 != value)
{
_logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}",
ImporterType, data[0][i]);
return false;
}
}
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
_logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
private void PerformImport(Layer importWorker)
{
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
ImporterType, SheetId, SheetTabName, DataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>();
try
{
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
if (data != null && data.Count >= 2)
{
for (var i = 0; i < data[1].Count; i++)
{
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
ImporterType, i);
continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
}
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
ImporterType, newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
ImporterType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
}