diff --git a/Utils/getProductionDatabase.sh b/Utils/getProductionDatabase.sh index a0be17f..6e229e1 100644 --- a/Utils/getProductionDatabase.sh +++ b/Utils/getProductionDatabase.sh @@ -13,7 +13,7 @@ DOCKER_SA_PASSWORD="$&#ojoOOKEJ223" DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd" DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE} #SERVER VARIABLES -REMOTE_HOST="crm.bim-it.pl" +REMOTE_HOST="bim-it.pl" REMOTE_USER="mz" REMOTE_SA_PASSWORD="v](8Lc|RfG" REMOTE_BACKUP_DIR="/tmp" diff --git a/Utils/totalCleanup.sh b/Utils/totalCleanup.sh new file mode 100644 index 0000000..f77ab94 --- /dev/null +++ b/Utils/totalCleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +docker stop $(docker ps -aq) +docker rm -f $(docker ps -aq) +docker rmi -f $(docker images -q) +docker network prune -f +docker volume prune -f +docker builder prune -af \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Models/Record.cs b/src/Backend/DiunaBI.Core/Models/Record.cs index f791f06..a554c56 100644 --- a/src/Backend/DiunaBI.Core/Models/Record.cs +++ b/src/Backend/DiunaBI.Core/Models/Record.cs @@ -45,7 +45,7 @@ public class Record public double? Value32 { get; set; } //Description fields [StringLength(10000)] - public string? Desc1 { get; init; } + public string? Desc1 { get; set; } public DateTime CreatedAt { get; set; } public DateTime ModifiedAt { get; set; } public bool IsDeleted { get; init; } diff --git a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs index 46c7b0c..dcb3c8d 100644 --- a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs +++ b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs @@ -4,6 +4,7 @@ using System.Text.RegularExpressions; using DiunaBI.Core.Models; namespace DiunaBI.Core.Services; + public static class ProcessHelper { public static void SetValue(Record record, int number, double? value) @@ -184,7 +185,6 @@ public static class ProcessHelper } return null; } - public static string GetSheetName(int month, int year) { if (month < 1 || month > 12) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 322cba4..d509374 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD1Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD1"; + public override string ImporterType => "Morska.Import.D1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 59f5afb..0050947 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD3Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD3"; + public override string ImporterType => "Morska.Import.D3"; private readonly AppDbContext _db; private readonly ILogger _logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 62c9de0..8b25067 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaFk2Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaFK2"; + public override string ImporterType => "Morska.Import.FK2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs index b655f15..4f8ac22 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -8,12 +9,23 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaImporter : MorskaBaseImporter { - public override string ImporterType => "MorskaImporter"; - + public override string ImporterType => "Morska.Import.Standard"; + private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaImporter( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,47 +38,193 @@ public class MorskaImporter : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + // Load configuration from layer records + LoadConfiguration(importWorker); + + // Check if import should be performed + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual import + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + // Check date range if configured + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + // Outside date range - check if imported layer is up to date + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + // No date constraints - always import + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; + + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; // Assume up to date if no data + } + + // Check if the number of columns matches + if (data[0].Count != data[1].Count) + { + _logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + for (var i = 0; i < data[1].Count; i++) + { + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); + continue; + } + + // Check if the record exists in the database - add null check + var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); + if (existingRecord == null || existingRecord.Value1 != value) + { + _logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}", + ImporterType, data[0][i]); + return false; + } + } + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - _logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + _logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { @@ -79,25 +237,29 @@ public class MorskaImporter : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get( + SheetId!, + $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); if (data != null && data.Count >= 2) { for (var i = 0; i < data[1].Count; i++) { - if (!(data[0][i].ToString()?.Length > 0) || - !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) { - _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i); + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); continue; } @@ -114,16 +276,16 @@ public class MorskaImporter : MorskaBaseImporter } _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } @@ -146,6 +308,12 @@ public class MorskaImporter : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 0a35f32..e26e973 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -20,15 +20,15 @@ public class LayersController : Controller private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; - private readonly ILogger _logger; + private readonly ILogger _logger; public LayersController( AppDbContext db, - SpreadsheetsResource.ValuesResource? googleSheetValues, + SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, - ILogger logger + ILogger logger ) { _db = db; @@ -59,7 +59,7 @@ public class LayersController : Controller .OrderByDescending(x => x.Number) .Skip(start).Take(limit).AsNoTracking().ToList(); - _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", + _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", result.Count, name, type); return Ok(result); @@ -228,7 +228,7 @@ public class LayersController : Controller _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); export.Export(layer); _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); - + return Ok(true); } catch (Exception e) @@ -276,7 +276,7 @@ public class LayersController : Controller } catch (Exception e) { - _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", + _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } @@ -328,7 +328,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", + _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", nameFilter, importWorkerLayers.Count); try @@ -345,8 +345,8 @@ public class LayersController : Controller { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; - - _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", + + _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", importWorker.Name, type, source); if (source == "DataInbox" && type == "Import-D3") @@ -357,8 +357,8 @@ public class LayersController : Controller throw new Exception("MorskaD3 importer not found"); } d3Importer.Import(importWorker); - - _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", + + _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); continue; } @@ -374,7 +374,7 @@ public class LayersController : Controller d1importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -387,7 +387,7 @@ public class LayersController : Controller fk2importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -406,7 +406,7 @@ public class LayersController : Controller var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - + if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = _pluginManager.GetImporter("MorskaImporter"); @@ -417,7 +417,7 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else if (IsImportedLayerUpToDate(importWorker) == false) @@ -430,12 +430,12 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", + _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else { - _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", + _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", importWorker.Name, importWorker.Id); } break; @@ -443,7 +443,7 @@ public class LayersController : Controller } catch (Exception e) { - _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", + _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } @@ -505,7 +505,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", + _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", type, processWorkerLayers.Count); foreach (var processWorker in processWorkerLayers) @@ -513,12 +513,12 @@ public class LayersController : Controller try { ProcessLayer(processWorker); - _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } catch (Exception e) { - _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } } @@ -674,6 +674,112 @@ public class LayersController : Controller } } + [HttpGet] + [Route("GetImportWorkers")] + [AllowAnonymous] + public IActionResult GetImportWorkers() + { + var importWorkerLayers = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + && x.Number == 3487 + ) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + foreach (var importWorker in importWorkerLayers) + { + _logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; + if (pluginName != null) + { + var importer = _pluginManager.GetImporter(pluginName); + if (importer == null) + { + _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})", + pluginName, importWorker.Name, importWorker.Id); + throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}"); + } + try + { + _logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); + importer.Import(importWorker); + _logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + } + catch (Exception e) + { + _logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); + throw; + } + } + else + { + _logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + throw new Exception($"No plugin name found for import worker layer {importWorker.Name}"); + } + } + return Ok(); + } + + [HttpGet] + [Route("AddPluginName")] + [AllowAnonymous] + public IActionResult AddPluginName() + { + var record = new Record + { + Id = Guid.NewGuid(), + LayerId = Guid.Parse("eb5b4d0e-1607-4445-bbe5-65b9b8416787"), + Code = "Plugin", + Desc1 = "Morska.Import.Standard", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + _db.Records.Add(record); + _db.SaveChanges(); + + /* + var importWorkerLayers = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") + ) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + foreach (var importWorker in importWorkerLayers) + { + var record = new Record + { + Id = Guid.NewGuid(), + LayerId = importWorker.Id, + Code = "Plugin", + Desc1 = "Morska.Import.D3", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + //_db.Records.Add(record); + } + + //_db.SaveChanges(); + */ + return Ok(); + } + private static void WriteToConsole(params string[] messages) { @@ -734,7 +840,7 @@ public class LayersController : Controller var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", data[0][i].ToString(), importWorker.Name); isUpToDate = false; continue; @@ -752,12 +858,12 @@ public class LayersController : Controller continue; } - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", record.Code, importWorker.Name); isUpToDate = false; } - _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", + _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 0d62ca9..43dcced 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -125,7 +125,9 @@ app.Use(async (context, next) => if (token.Length > 0 && !context.Request.Path.ToString().Contains("getForPowerBI") && !context.Request.Path.ToString().Contains("getConfiguration") - && !context.Request.Path.ToString().Contains("DataInbox/Add")) + && !context.Request.Path.ToString().Contains("DataInbox/Add") + && !context.Request.Path.ToString().Contains("AddPluginName") // TODO: Remove this + && !context.Request.Path.ToString().Contains("GetImportWorkers")) { var handler = new JwtSecurityTokenHandler(); var data = handler.ReadJwtToken(token.Split(' ')[1]); diff --git a/src/Frontend/src/environments/environment.ts b/src/Frontend/src/environments/environment.ts index 98dcec0..b4ed8cc 100644 --- a/src/Frontend/src/environments/environment.ts +++ b/src/Frontend/src/environments/environment.ts @@ -7,8 +7,8 @@ export const environment = { appName: "LOCAL_DiunaBI", production: false, api: { - //url: "http://localhost:5400/api" - url: "https://diunabi-morska.bim-it.pl/api" + url: "http://localhost:5400/api" + //url: "https://diunabi-morska.bim-it.pl/api" }, google: { clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com" diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index 784b5ed..53b13da 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5- +GET http://localhost:5400/api/Layers/GetImportWorkers