Morska.Import.Standard refactored
This commit is contained in:
Michał Zieliński
@@ -1,6 +1,7 @@
|
||||
using System.Globalization;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using DiunaBI.Core.Models;
|
||||
using DiunaBI.Database.Context;
|
||||
|
||||
@@ -8,12 +9,23 @@ namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaImporter : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "MorskaImporter";
|
||||
|
||||
public override string ImporterType => "Morska.Import.Standard";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaImporter> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
public MorskaImporter(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
@@ -26,47 +38,193 @@ public class MorskaImporter : MorskaBaseImporter
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
importWorker.Name, importWorker.Id);
|
||||
|
||||
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
|
||||
if (sheetId == null)
|
||||
try
|
||||
{
|
||||
throw new Exception($"SheetId not found, {importWorker.Name}");
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
// Check if import should be performed
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual import
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
|
||||
if (sheetTabName == null)
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
throw new Exception($"SheetTabName not found, {importWorker.Name}");
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||
if (year == null)
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||
if (month == null)
|
||||
// Check date range if configured
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Outside date range - check if imported layer is up to date
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||
if (name == null)
|
||||
// No date constraints - always import
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
|
||||
if (dataRange == null)
|
||||
try
|
||||
{
|
||||
throw new Exception($"DataRange not found, {importWorker.Name}");
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
var data = dataRangeResponse.Values;
|
||||
|
||||
if (data == null || data.Count < 2)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true; // Assume up to date if no data
|
||||
}
|
||||
|
||||
// Check if the number of columns matches
|
||||
if (data[0].Count != data[1].Count)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if the record exists in the database - add null check
|
||||
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
|
||||
if (existingRecord == null || existingRecord.Value1 != value)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}",
|
||||
ImporterType, data[0][i]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
sheetId, sheetTabName, dataRange);
|
||||
_logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
@@ -79,25 +237,29 @@ public class MorskaImporter : MorskaBaseImporter
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
|
||||
var dataRangeResponse = _googleSheetValues.Get(
|
||||
SheetId!,
|
||||
$"{SheetTabName}!{DataRange}").Execute();
|
||||
var data = dataRangeResponse.Values;
|
||||
|
||||
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
|
||||
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null && data.Count >= 2)
|
||||
{
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (!(data[0][i].ToString()?.Length > 0) ||
|
||||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -114,16 +276,16 @@ public class MorskaImporter : MorskaBaseImporter
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
newRecords.Count, layer.Name, layer.Id);
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -146,6 +308,12 @@ public class MorskaImporter : MorskaBaseImporter
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user