after refactor cleanup
This commit is contained in:
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
@@ -0,0 +1,397 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaFk2Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.FK2";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaFk2Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cache for sheet data
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaFk2Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaFk2Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// ✅ Clear cache at start
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// ✅ Clear cache after import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ✅ Dodaj metodę cache
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var isUpToDate = true;
|
||||
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue;
|
||||
|
||||
try
|
||||
{
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3) continue;
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||
if (record == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||
ImporterType, code, importWorker.Name);
|
||||
isUpToDate = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) &&
|
||||
Math.Abs((double)(record.Value1 - value)!) >= 0.01)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, record.Value1, value);
|
||||
isUpToDate = false;
|
||||
}
|
||||
|
||||
if (record.Desc1 != data[i][3]?.ToString())
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}",
|
||||
ImporterType, code, record.Desc1, data[i][3]?.ToString());
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison",
|
||||
ImporterType, i);
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null)
|
||||
{
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}",
|
||||
ImporterType, i, data[i][1]);
|
||||
throw new InvalidOperationException($"Invalid date in row {i}");
|
||||
}
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
||||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = code,
|
||||
Desc1 = data[i][3].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
newRecords.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user