mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor processors

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-07 13:51:27 +02:00
parent 787df8b838
commit d05dc34e97
14 changed files with 2996 additions and 917 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
View File

@@ -26,6 +26,10 @@ public class MorskaD1Importer : MorskaBaseImporter
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaD1Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -43,6 +47,10 @@ public class MorskaD1Importer : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// ✅ Clear cache at start
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker))
@@ -65,6 +73,46 @@ public class MorskaD1Importer : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// ✅ Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
// ✅ Dodaj metodę cache
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
@@ -164,8 +212,8 @@ public class MorskaD1Importer : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
if (data == null || data.Count < 2)
{
@@ -239,10 +287,10 @@ public class MorskaD1Importer : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet",
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
var newRecords = (from t in data
@@ -279,8 +327,7 @@ public class MorskaD1Importer : MorskaBaseImporter
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}",
ImporterType, SheetId);
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}

61
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
View File

@@ -26,6 +26,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaFk2Importer(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -43,6 +47,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// ✅ Clear cache at start
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker))
@@ -65,6 +73,46 @@ public class MorskaFk2Importer : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// ✅ Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
// ✅ Dodaj metodę cache
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
@@ -164,8 +212,8 @@ public class MorskaFk2Importer : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
if (data == null || data.Count == 0)
{
@@ -256,10 +304,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
// ✅ Użyj cache zamiast bezpośredniego API
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet",
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
if (data != null)
@@ -315,8 +363,7 @@ public class MorskaFk2Importer : MorskaBaseImporter
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}",
ImporterType, SheetId);
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}

70
src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs → src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs
View File

@@ -7,13 +7,13 @@ using DiunaBI.Core.Database.Context;
namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaImporter : MorskaBaseImporter
public class MorskaStandardImporter : MorskaBaseImporter
{
public override string ImporterType => "Morska.Import.Standard";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaImporter> _logger;
private readonly ILogger<MorskaStandardImporter> _logger;
// Configuration properties
private string? SheetId { get; set; }
@@ -26,10 +26,13 @@ public class MorskaImporter : MorskaBaseImporter
private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; }
public MorskaImporter(
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaStandardImporter(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaImporter> logger)
ILogger<MorskaStandardImporter> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
@@ -43,6 +46,10 @@ public class MorskaImporter : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id);
// Clear cache before import
_cachedSheetData = null;
_cachedDataKey = null;
// Load configuration from layer records
LoadConfiguration(importWorker);
@@ -69,6 +76,45 @@ public class MorskaImporter : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id);
throw;
}
finally
{
// Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
}
private void LoadConfiguration(Layer importWorker)
@@ -171,14 +217,13 @@ public class MorskaImporter : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
var data = GetSheetData();
if (data == null || data.Count < 2)
{
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name);
return true; // Assume up to date if no data
return true;
}
// Check if the number of columns matches
@@ -199,7 +244,6 @@ public class MorskaImporter : MorskaBaseImporter
continue;
}
// Check if the record exists in the database - add null check
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (existingRecord == null || existingRecord.Value1 != value)
{
@@ -243,12 +287,9 @@ public class MorskaImporter : MorskaBaseImporter
try
{
var dataRangeResponse = _googleSheetValues.Get(
SheetId!,
$"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet",
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0);
if (data != null && data.Count >= 2)
@@ -284,8 +325,7 @@ public class MorskaImporter : MorskaBaseImporter
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}",
ImporterType, SheetId);
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
throw;
}
}