after refactor cleanup
This commit is contained in:
13
DiunaBI.Plugins.Morska/Importers/MorskaBaseImporter.cs
Normal file
13
DiunaBI.Plugins.Morska/Importers/MorskaBaseImporter.cs
Normal file
@@ -0,0 +1,13 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Interfaces;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public abstract class MorskaBaseImporter : IDataImporter
|
||||
{
|
||||
public abstract string ImporterType { get; }
|
||||
|
||||
public virtual bool CanImport(string importerType) => ImporterType == importerType;
|
||||
|
||||
public abstract void Import(Layer importWorker);
|
||||
}
|
||||
415
DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
Normal file
415
DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
Normal file
@@ -0,0 +1,415 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaD1Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.D1";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaD1Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cache for sheet data
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaD1Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaD1Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, need to create new layer",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count < 2)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var isUpToDate = true;
|
||||
|
||||
foreach (var row in data)
|
||||
{
|
||||
if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
|
||||
|
||||
var code = row[0].ToString();
|
||||
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||
|
||||
if (record == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||
ImporterType, code, importWorker.Name);
|
||||
isUpToDate = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check values 3-17 (Value1-Value15)
|
||||
for (int i = 3; i <= 17 && i < row.Count; i++)
|
||||
{
|
||||
var sheetValue = ParseValue(row[i]?.ToString());
|
||||
var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
|
||||
|
||||
if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, i - 2, dbValue, sheetValue);
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
// check also Desc1 from records (Value18 in GSheet)
|
||||
if (row.Count <= 18) continue;
|
||||
var sheetDesc1 = row[18]?.ToString();
|
||||
var recordDesc1 = record.Desc1;
|
||||
|
||||
var normalizedSheetDesc1 = string.IsNullOrEmpty(sheetDesc1) ? null : sheetDesc1;
|
||||
var normalizedRecordDesc1 = string.IsNullOrEmpty(recordDesc1) ? null : recordDesc1;
|
||||
|
||||
if (normalizedSheetDesc1 == normalizedRecordDesc1) continue;
|
||||
_logger.LogDebug("{ImporterType}: Desc1 mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, normalizedRecordDesc1, normalizedSheetDesc1);
|
||||
isUpToDate = false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
var newRecords = (from t in data
|
||||
where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
|
||||
select new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = t[0].ToString(),
|
||||
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
|
||||
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
|
||||
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
|
||||
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
|
||||
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
|
||||
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
|
||||
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
|
||||
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
|
||||
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
|
||||
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
|
||||
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
|
||||
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
|
||||
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
|
||||
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
|
||||
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
|
||||
Desc1 = IndexExists(t, 18) ? t[18]?.ToString() : null,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
}).ToList();
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private double? ParseValue(string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
|
||||
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
|
||||
try
|
||||
{
|
||||
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
||||
return result;
|
||||
}
|
||||
catch (FormatException e)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}",
|
||||
ImporterType, value, e.Message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private bool IndexExists(IList<object> array, int index)
|
||||
{
|
||||
return array != null && index >= 0 && index < array.Count;
|
||||
}
|
||||
|
||||
private double? GetRecordValueByIndex(Record record, int valueIndex)
|
||||
{
|
||||
return valueIndex switch
|
||||
{
|
||||
1 => record.Value1,
|
||||
2 => record.Value2,
|
||||
3 => record.Value3,
|
||||
4 => record.Value4,
|
||||
5 => record.Value5,
|
||||
6 => record.Value6,
|
||||
7 => record.Value7,
|
||||
8 => record.Value8,
|
||||
9 => record.Value9,
|
||||
10 => record.Value10,
|
||||
11 => record.Value11,
|
||||
12 => record.Value12,
|
||||
13 => record.Value13,
|
||||
14 => record.Value14,
|
||||
15 => record.Value15,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
428
DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs
Normal file
428
DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs
Normal file
@@ -0,0 +1,428 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaD3Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.D3";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaD3Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private string? ImportType { get; set; }
|
||||
private string? SheetId { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cached deserialized data
|
||||
private List<Record>? _cachedRecords;
|
||||
private string? _cachedDataInboxId;
|
||||
|
||||
public MorskaD3Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaD3Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Clear cache at start
|
||||
_cachedRecords = null;
|
||||
_cachedDataInboxId = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
// Deserialize data early - right after LoadConfiguration
|
||||
DeserializeDataInboxData();
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
// Export to Google Sheets after successful import
|
||||
ExportToGoogleSheets();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clear cache after import
|
||||
_cachedRecords = null;
|
||||
_cachedDataInboxId = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
ImportType = GetRecordValue(importWorker.Records, "ImportType");
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}, SheetId: {SheetId}",
|
||||
ImporterType, importWorker.Name, ImportType, SheetId);
|
||||
}
|
||||
|
||||
private void DeserializeDataInboxData()
|
||||
{
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||
ImporterType, dataInbox.Id, dataInbox.CreatedAt);
|
||||
|
||||
try
|
||||
{
|
||||
var data = Convert.FromBase64String(dataInbox.Data);
|
||||
var jsonString = Encoding.UTF8.GetString(data);
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
|
||||
ImporterType, data.Length);
|
||||
|
||||
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
||||
if (records == null)
|
||||
{
|
||||
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
|
||||
ImporterType, records.Count);
|
||||
|
||||
// Cache the deserialized data
|
||||
_cachedRecords = records;
|
||||
_cachedDataInboxId = dataInbox.Id.ToString();
|
||||
}
|
||||
catch (FormatException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxName}",
|
||||
ImporterType, ImportType);
|
||||
throw new InvalidOperationException($"Invalid base64 data in DataInbox: {ImportType}", e);
|
||||
}
|
||||
catch (JsonException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxName}",
|
||||
ImporterType, ImportType);
|
||||
throw new InvalidOperationException($"Invalid JSON data in DataInbox: {ImportType}", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Record> GetFilteredRecords()
|
||||
{
|
||||
if (_cachedRecords == null)
|
||||
{
|
||||
throw new InvalidOperationException("Data has not been deserialized yet");
|
||||
}
|
||||
|
||||
var filteredRecords = _cachedRecords.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
|
||||
if (filteredRecords.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||
ImporterType, filteredRecords.Count, ImportYear, ImportMonth);
|
||||
|
||||
return filteredRecords;
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No DataInbox found for type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if imported layer is up to date
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x =>
|
||||
x.ParentId == importWorker.Id
|
||||
&& x.Name!.Contains($"I-{ImportName}-{ImportYear}/{ImportMonth}"))
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, import needed",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var currentRecords = GetFilteredRecords();
|
||||
|
||||
// Compare record counts first
|
||||
if (newestLayer.Records?.Count != currentRecords.Count)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Record count mismatch - DB: {DbCount}, DataInbox: {DataCount}",
|
||||
ImporterType, newestLayer.Records?.Count ?? 0, currentRecords.Count);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare individual records
|
||||
foreach (var currentRecord in currentRecords)
|
||||
{
|
||||
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == currentRecord.Code);
|
||||
if (existingRecord == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Record with code {Code} not found in existing layer",
|
||||
ImporterType, currentRecord.Code);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare all relevant fields
|
||||
if (Math.Abs((existingRecord.Value1 ?? 0) - (currentRecord.Value1 ?? 0)) > 0.001 ||
|
||||
existingRecord.Desc1 != currentRecord.Desc1)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Data difference found for code {Code}",
|
||||
ImporterType, currentRecord.Code);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: All records match, layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Starting import for DataInbox type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
|
||||
var filteredRecords = GetFilteredRecords();
|
||||
|
||||
// Prepare records for saving
|
||||
var recordsToSave = filteredRecords.Select(x =>
|
||||
{
|
||||
x.Id = Guid.NewGuid();
|
||||
x.CreatedAt = DateTime.UtcNow;
|
||||
x.ModifiedAt = DateTime.UtcNow;
|
||||
return x;
|
||||
}).ToList();
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, recordsToSave);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation(
|
||||
"{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, recordsToSave.Count, layer.Name, layer.Id);
|
||||
}
|
||||
|
||||
private void ExportToGoogleSheets()
|
||||
{
|
||||
if (string.IsNullOrEmpty(SheetId))
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: SheetId not configured, skipping Google Sheets export",
|
||||
ImporterType);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Starting Google Sheets export to {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
|
||||
var filteredRecords = GetFilteredRecords();
|
||||
var sheetTabName = ProcessHelper.GetSheetName(int.Parse(ImportMonth!), int.Parse(ImportYear!));
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using sheet tab name: {SheetTabName}",
|
||||
ImporterType, sheetTabName);
|
||||
|
||||
// Get current sheet data
|
||||
var currentSheetData = _googleSheetValues.Get(SheetId!, $"{sheetTabName}!C7:D200").Execute();
|
||||
|
||||
var updateRequests = new List<ValueRange>();
|
||||
|
||||
for (var rowIndex = 0; rowIndex < 194; rowIndex++)
|
||||
{
|
||||
if (currentSheetData.Values == null || rowIndex >= currentSheetData.Values.Count)
|
||||
continue;
|
||||
|
||||
var existingRow = currentSheetData.Values[rowIndex];
|
||||
if (existingRow.Count == 0 || string.IsNullOrEmpty(existingRow[0]?.ToString()))
|
||||
continue;
|
||||
|
||||
var accountCode = existingRow[0].ToString()!;
|
||||
var matchingRecord = filteredRecords.FirstOrDefault(x => x.Desc1 == accountCode);
|
||||
if (matchingRecord == null)
|
||||
continue;
|
||||
|
||||
var newValue = matchingRecord.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0";
|
||||
var existingValue = (existingRow.Count > 1 ? existingRow[1] : "")?.ToString();
|
||||
|
||||
if (existingValue != newValue)
|
||||
{
|
||||
var range = $"{sheetTabName}!D{rowIndex + 7}";
|
||||
updateRequests.Add(new ValueRange
|
||||
{
|
||||
Range = range,
|
||||
Values = new List<IList<object>> { new List<object> { newValue } }
|
||||
});
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Scheduled update for {AccountCode} at {Range} with value {Value}",
|
||||
ImporterType, accountCode, range, newValue);
|
||||
}
|
||||
}
|
||||
|
||||
if (updateRequests.Count > 0)
|
||||
{
|
||||
var batchUpdate = new BatchUpdateValuesRequest
|
||||
{
|
||||
Data = updateRequests,
|
||||
ValueInputOption = "USER_ENTERED"
|
||||
};
|
||||
|
||||
_googleSheetValues.BatchUpdate(batchUpdate, SheetId!).Execute();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Updated {Count} cells in Google Sheet {SheetId}",
|
||||
ImporterType, updateRequests.Count, SheetId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: No changes to export to Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to export to Google Sheets {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
@@ -0,0 +1,397 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaFk2Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.FK2";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaFk2Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cache for sheet data
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaFk2Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaFk2Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// ✅ Clear cache at start
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// ✅ Clear cache after import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ✅ Dodaj metodę cache
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var isUpToDate = true;
|
||||
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue;
|
||||
|
||||
try
|
||||
{
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3) continue;
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||
if (record == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||
ImporterType, code, importWorker.Name);
|
||||
isUpToDate = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) &&
|
||||
Math.Abs((double)(record.Value1 - value)!) >= 0.01)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, record.Value1, value);
|
||||
isUpToDate = false;
|
||||
}
|
||||
|
||||
if (record.Desc1 != data[i][3]?.ToString())
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}",
|
||||
ImporterType, code, record.Desc1, data[i][3]?.ToString());
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison",
|
||||
ImporterType, i);
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null)
|
||||
{
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}",
|
||||
ImporterType, i, data[i][1]);
|
||||
throw new InvalidOperationException($"Invalid date in row {i}");
|
||||
}
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
||||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = code,
|
||||
Desc1 = data[i][3].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
newRecords.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
360
DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs
Normal file
360
DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs
Normal file
@@ -0,0 +1,360 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaStandardImporter : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.Standard";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaStandardImporter> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaStandardImporter(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaStandardImporter> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Clear cache before import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
// Check if import should be performed
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual import
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clear cache after import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check date range if configured
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Outside date range - check if imported layer is up to date
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// No date constraints - always import
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count < 2)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check if the number of columns matches
|
||||
if (data[0].Count != data[1].Count)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
|
||||
if (existingRecord == null || existingRecord.Value1 != value)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}",
|
||||
ImporterType, data[0][i]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null && data.Count >= 2)
|
||||
{
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = data[0][i].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
newRecords.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user