mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Morska.Import.D1 refactored

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-06 20:43:19 +02:00
parent b73adb4b17
commit ee9307c7c3
2 changed files with 243 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
View File

@@ -1,6 +1,7 @@
using System.Globalization;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
@@ -14,6 +15,17 @@ public class MorskaD1Importer : MorskaBaseImporter
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD1Importer> _logger;
// Configuration properties
private string? SheetId { get; set; }
private string? SheetTabName { get; set; }
private string? DataRange { get; set; }
private string? ImportYear { get; set; }
private string? ImportMonth { get; set; }
private string? ImportName { get; set; }
private DateTime? StartDate { get; set; }
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
public MorskaD1Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -26,51 +38,195 @@ public class MorskaD1Importer : MorskaBaseImporter
public override void Import(Layer importWorker)
{
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null)
try
{
throw new Exception($"SheetId not found, {importWorker.Name}");
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker))
{
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return;
}
ValidateConfiguration();
PerformImport(importWorker);
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
{
if (importWorker.Records == null) return;
SheetId = GetRecordValue(importWorker.Records, "SheetId");
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
DataRange = GetRecordValue(importWorker.Records, "DataRange");
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
ImportName = GetRecordValue(importWorker.Records, "ImportName");
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
{
StartDate = startDate;
}
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null)
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
EndDate = endDate;
}
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null)
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
ImporterType, importWorker.Name);
}
private bool ShouldPerformImport(Layer importWorker)
{
if (!IsEnabled)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null)
if (StartDate.HasValue && EndDate.HasValue)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
var now = DateTime.UtcNow.Date;
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
{
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
if (!IsImportedLayerUpToDate(importWorker))
{
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
ImporterType, importWorker.Name);
return false;
}
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null)
return true;
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
if (errors.Any())
{
throw new Exception($"ImportName not found, {importWorker.Name}");
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
}
private bool IsImportedLayerUpToDate(Layer importWorker)
{
var newestLayer = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == importWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.FirstOrDefault();
if (newestLayer == null)
{
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
ImporterType, importWorker.Name);
return true;
}
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
if (dataRange == null)
try
{
throw new Exception($"DataRange not found, {importWorker.Name}");
}
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
if (data == null || data.Count < 2)
{
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true;
}
var isUpToDate = true;
foreach (var row in data)
{
if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
var code = row[0].ToString();
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
if (record == null)
{
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
ImporterType, code, importWorker.Name);
isUpToDate = false;
continue;
}
// Check values 3-17 (Value1-Value15)
for (int i = 3; i <= 17 && i < row.Count; i++)
{
var sheetValue = ParseValue(row[i]?.ToString());
var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
{
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}",
ImporterType, code, i - 2, dbValue, sheetValue);
isUpToDate = false;
}
}
}
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
return isUpToDate;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
ImporterType, importWorker.Name);
throw;
}
}
private void PerformImport(Layer importWorker)
{
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
ImporterType, SheetId, SheetTabName, DataRange);
var layer = new Layer
{
Id = Guid.NewGuid(),
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
@@ -79,17 +235,18 @@ public class MorskaD1Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
try
{
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet",
ImporterType, data?.Count ?? 0);
var newRecords = (from t in data
where t.Count > 1 && (string)t[0] != string.Empty
where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
select new Record
{
Id = Guid.NewGuid(),
@@ -114,20 +271,70 @@ public class MorskaD1Importer : MorskaBaseImporter
}).ToList();
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
ImporterType, newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private double? ParseValue(string? value)
{
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
try
{
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result;
}
catch (FormatException e)
{
_logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}",
ImporterType, value, e.Message);
return null;
}
}
private bool IndexExists(IList<object> array, int index)
{
return array != null && index >= 0 && index < array.Count;
}
private double? GetRecordValueByIndex(Record record, int valueIndex)
{
return valueIndex switch
{
1 => record.Value1,
2 => record.Value2,
3 => record.Value3,
4 => record.Value4,
5 => record.Value5,
6 => record.Value6,
7 => record.Value7,
8 => record.Value8,
9 => record.Value9,
10 => record.Value10,
11 => record.Value11,
12 => record.Value12,
13 => record.Value13,
14 => record.Value14,
15 => record.Value15,
_ => null
};
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
@@ -146,27 +353,7 @@ public class MorskaD1Importer : MorskaBaseImporter
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private double? ParseValue(string? value)
{
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
try
{
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result;
}
catch (FormatException e)
{
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
return null;
}
}
private bool IndexExists(IList<object> array, int index)
{
return array != null && index >= 0 && index < array.Count;
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
ImporterType, records.Count, layerId);
}
}

2
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -684,7 +684,7 @@ public class LayersController : Controller
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
&& x.Number == 3487
&& x.Number == 8215
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()