mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Morska.Import.Standard refactored

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-06 20:23:36 +02:00
parent a4d8622ace
commit b73adb4b17
12 changed files with 354 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -20,15 +20,15 @@ public class LayersController : Controller
private readonly GoogleDriveHelper _googleDriveHelper;
private readonly IConfiguration _configuration;
private readonly PluginManager _pluginManager;
private readonly ILogger<LayersController> _logger;
private readonly ILogger<LayersController> _logger;
public LayersController(
AppDbContext db,
SpreadsheetsResource.ValuesResource? googleSheetValues,
SpreadsheetsResource.ValuesResource? googleSheetValues,
GoogleDriveHelper googleDriveHelper,
IConfiguration configuration,
PluginManager pluginManager,
ILogger<LayersController> logger
ILogger<LayersController> logger
)
{
_db = db;
@@ -59,7 +59,7 @@ public class LayersController : Controller
.OrderByDescending(x => x.Number)
.Skip(start).Take(limit).AsNoTracking().ToList();
_logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}",
_logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}",
result.Count, name, type);
return Ok(result);
@@ -228,7 +228,7 @@ public class LayersController : Controller
_logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name);
export.Export(layer);
_logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id);
return Ok(true);
}
catch (Exception e)
@@ -276,7 +276,7 @@ public class LayersController : Controller
}
catch (Exception e)
{
_logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})",
_logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
}
@@ -328,7 +328,7 @@ public class LayersController : Controller
.AsNoTracking()
.ToList();
_logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers",
_logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers",
nameFilter, importWorkerLayers.Count);
try
@@ -345,8 +345,8 @@ public class LayersController : Controller
{
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard";
var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet";
_logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}",
_logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}",
importWorker.Name, type, source);
if (source == "DataInbox" && type == "Import-D3")
@@ -357,8 +357,8 @@ public class LayersController : Controller
throw new Exception("MorskaD3 importer not found");
}
d3Importer.Import(importWorker);
_logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})",
_logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
continue;
}
@@ -374,7 +374,7 @@ public class LayersController : Controller
d1importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})",
_logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
break;
@@ -387,7 +387,7 @@ public class LayersController : Controller
fk2importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})",
_logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
break;
@@ -406,7 +406,7 @@ public class LayersController : Controller
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
var importer = _pluginManager.GetImporter("MorskaImporter");
@@ -417,7 +417,7 @@ public class LayersController : Controller
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})",
_logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
else if (IsImportedLayerUpToDate(importWorker) == false)
@@ -430,12 +430,12 @@ public class LayersController : Controller
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})",
_logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
else
{
_logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping",
_logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping",
importWorker.Name, importWorker.Id);
}
break;
@@ -443,7 +443,7 @@ public class LayersController : Controller
}
catch (Exception e)
{
_logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})",
_logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
}
@@ -505,7 +505,7 @@ public class LayersController : Controller
.AsNoTracking()
.ToList();
_logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers",
_logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers",
type, processWorkerLayers.Count);
foreach (var processWorker in processWorkerLayers)
@@ -513,12 +513,12 @@ public class LayersController : Controller
try
{
ProcessLayer(processWorker);
_logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}",
_logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}",
processWorker.Name, processWorker.Id, type);
}
catch (Exception e)
{
_logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}",
_logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}",
processWorker.Name, processWorker.Id, type);
}
}
@@ -674,6 +674,112 @@ public class LayersController : Controller
}
}
[HttpGet]
[Route("GetImportWorkers")]
[AllowAnonymous]
public IActionResult GetImportWorkers()
{
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
&& x.Number == 3487
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
foreach (var importWorker in importWorkerLayers)
{
_logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (pluginName != null)
{
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})",
pluginName, importWorker.Name, importWorker.Id);
throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}");
}
try
{
_logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
importer.Import(importWorker);
_logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
catch (Exception e)
{
_logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
throw;
}
}
else
{
_logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
throw new Exception($"No plugin name found for import worker layer {importWorker.Name}");
}
}
return Ok();
}
[HttpGet]
[Route("AddPluginName")]
[AllowAnonymous]
public IActionResult AddPluginName()
{
var record = new Record
{
Id = Guid.NewGuid(),
LayerId = Guid.Parse("eb5b4d0e-1607-4445-bbe5-65b9b8416787"),
Code = "Plugin",
Desc1 = "Morska.Import.Standard",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
_db.Records.Add(record);
_db.SaveChanges();
/*
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
foreach (var importWorker in importWorkerLayers)
{
var record = new Record
{
Id = Guid.NewGuid(),
LayerId = importWorker.Id,
Code = "Plugin",
Desc1 = "Morska.Import.D3",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
//_db.Records.Add(record);
}
//_db.SaveChanges();
*/
return Ok();
}
private static void WriteToConsole(params string[] messages)
{
@@ -734,7 +840,7 @@ public class LayersController : Controller
var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (record == null)
{
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}",
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}",
data[0][i].ToString(), importWorker.Name);
isUpToDate = false;
continue;
@@ -752,12 +858,12 @@ public class LayersController : Controller
continue;
}
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}",
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}",
record.Code, importWorker.Name);
isUpToDate = false;
}
_logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}",
_logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}",
importWorker.Name, isUpToDate ? "up to date" : "outdated");
return isUpToDate;