WIP: p2 plugin
Some checks failed
Build Docker Images / test (map[name:Morska plugin_project:DiunaBI.Plugins.Morska]) (push) Failing after 1m14s
Build Docker Images / test (map[name:PedrolloPL plugin_project:DiunaBI.Plugins.PedrolloPL]) (push) Failing after 1m10s
Build Docker Images / build-and-push (map[image_suffix:morska name:Morska plugin_project:DiunaBI.Plugins.Morska]) (push) Failing after 1m12s
Build Docker Images / build-and-push (map[image_suffix:pedrollopl name:PedrolloPL plugin_project:DiunaBI.Plugins.PedrolloPL]) (push) Failing after 1m7s
Some checks failed
Build Docker Images / test (map[name:Morska plugin_project:DiunaBI.Plugins.Morska]) (push) Failing after 1m14s
Build Docker Images / test (map[name:PedrolloPL plugin_project:DiunaBI.Plugins.PedrolloPL]) (push) Failing after 1m10s
Build Docker Images / build-and-push (map[image_suffix:morska name:Morska plugin_project:DiunaBI.Plugins.Morska]) (push) Failing after 1m12s
Build Docker Images / build-and-push (map[image_suffix:pedrollopl name:PedrolloPL plugin_project:DiunaBI.Plugins.PedrolloPL]) (push) Failing after 1m7s
This commit is contained in:
99
DevTools/PedrolloImport.sql
Normal file
99
DevTools/PedrolloImport.sql
Normal file
@@ -0,0 +1,99 @@
|
||||
DECLARE @JustForDebug TINYINT = 0;
|
||||
|
||||
-- SETUP VARIABLES
|
||||
DECLARE @Year INT = 2024;
|
||||
DECLARE @Type NVARCHAR(5) = 'P2';
|
||||
DECLARE @StartDate NVARCHAR(10) = '2025.01.02';
|
||||
DECLARE @EndDate NVARCHAR(10) = '2026.12.31'
|
||||
|
||||
|
||||
DECLARE @Number INT = (SELECT COUNT(id) + 1 FROM [DiunaBI-PedrolloPL].[dbo].[Layers]);
|
||||
DECLARE @CurrentTimestamp NVARCHAR(14) = FORMAT(GETDATE(), 'yyyyMMddHHmm');
|
||||
DECLARE @Name NVARCHAR(50) = CONCAT(
|
||||
'L', @Number, '-A-IW_', @Type, '-', @Year,'-', @CurrentTimestamp
|
||||
);
|
||||
DECLARE @Plugin NVARCHAR(100);
|
||||
SET @Plugin =
|
||||
CASE @Type
|
||||
WHEN 'P2' THEN 'PedrolloPL.Import.P2'
|
||||
ELSE NULL -- If @Type doesn't match, set it to NULL
|
||||
END;
|
||||
|
||||
DECLARE @DataInboxName NVARCHAR(100);
|
||||
SET @DataInboxName =
|
||||
CASE @Type
|
||||
WHEN 'P2' THEN 'P2_2024'
|
||||
ELSE NULL -- If @Type doesn't match, set it to NULL
|
||||
END;
|
||||
|
||||
DECLARE @DataInboxSource NVARCHAR(100);
|
||||
SET @DataInboxSource =
|
||||
CASE @Type
|
||||
WHEN 'P2' THEN 'Comarch'
|
||||
ELSE NULL -- If @Type doesn't match, set it to NULL
|
||||
END;
|
||||
|
||||
|
||||
DECLARE @LayerId UNIQUEIDENTIFIER = NEWID();
|
||||
|
||||
SELECT @Name AS Name, @StartDate AS StartDate, @EndDate AS EndDate, @Type AS Type, @Year AS Year, @Plugin AS Plugin,
|
||||
@DataInboxName AS DataInboxName, @DataInboxSource AS DataInboxSource;
|
||||
|
||||
IF @JustForDebug = 1
|
||||
BEGIN
|
||||
SELECT 'Just for debug' AS Logger;
|
||||
RETURN;
|
||||
END;
|
||||
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Layers]
|
||||
([Id], [Number], [Name], [CreatedAt], [ModifiedAt], [IsDeleted], [IsCancelled], [CreatedById], [ModifiedById], [Type])
|
||||
VALUES (@LayerId, @Number, @Name, GETDATE(), GETDATE(), 0, 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'StartDate', @StartDate, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'EndDate', @EndDate, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'Source', 'DataInbox', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'ImportName', @Type, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'ImportYear', @Year, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'Type', 'ImportWorker', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'Plugin', @Plugin, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'IsEnabled', 'True', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'DataInboxName', @DataInboxName, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'DataInboxSource', @DataInboxSource, GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'Priority', '10', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
|
||||
INSERT INTO [DiunaBI-PedrolloPL].[dbo].[Records]
|
||||
([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
|
||||
VALUES ((SELECT NEWID()), 'MaxRetries', '3', GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, @LayerId);
|
||||
@@ -1,9 +1 @@
|
||||
PUT https://pedrollopl.diunabi.com/api/DataInbox/Add/8kL2mN4pQ6rojshf8704i34p4eim1hs
|
||||
Content-Type: application/json
|
||||
Authorization: Basic cGVkcm9sbG9wbDo0MjU4dlc2eFk4TjRwUQ==
|
||||
|
||||
{
|
||||
"Source": "morska.import",
|
||||
"Name": "morska.d3.importer",
|
||||
"Data": "eyJrZXkiOiAidmFsdWUifQ=="
|
||||
}
|
||||
POST http://localhost:5400/jobs/schedule/10763478CB738D4ecb2h76g803478CB738D4e
|
||||
@@ -320,4 +320,116 @@ public class JobsController : Controller
|
||||
return BadRequest(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
[HttpPost]
|
||||
[Route("create-for-layer/{layerId:guid}")]
|
||||
public async Task<IActionResult> CreateJobForLayer(Guid layerId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var layer = await _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.FirstOrDefaultAsync(l => l.Id == layerId);
|
||||
|
||||
if (layer == null)
|
||||
{
|
||||
_logger.LogWarning("CreateJobForLayer: Layer {LayerId} not found", layerId);
|
||||
return NotFound($"Layer {layerId} not found");
|
||||
}
|
||||
|
||||
if (layer.Type != LayerType.Administration)
|
||||
{
|
||||
_logger.LogWarning("CreateJobForLayer: Layer {LayerId} is not an Administration layer", layerId);
|
||||
return BadRequest("Only Administration layers can be run as jobs");
|
||||
}
|
||||
|
||||
// Get the Type record to determine if it's ImportWorker or ProcessWorker
|
||||
var typeRecord = layer.Records?.FirstOrDefault(x => x.Code == "Type");
|
||||
if (typeRecord?.Desc1 != "ImportWorker" && typeRecord?.Desc1 != "ProcessWorker")
|
||||
{
|
||||
_logger.LogWarning("CreateJobForLayer: Layer {LayerId} is not a valid worker type", layerId);
|
||||
return BadRequest("Layer must be an ImportWorker or ProcessWorker");
|
||||
}
|
||||
|
||||
// Check if enabled
|
||||
var isEnabledRecord = layer.Records?.FirstOrDefault(x => x.Code == "IsEnabled");
|
||||
if (isEnabledRecord?.Desc1 != "True")
|
||||
{
|
||||
_logger.LogWarning("CreateJobForLayer: Layer {LayerId} is not enabled", layerId);
|
||||
return BadRequest("Layer is not enabled");
|
||||
}
|
||||
|
||||
// Get plugin name
|
||||
var pluginRecord = layer.Records?.FirstOrDefault(x => x.Code == "Plugin");
|
||||
if (string.IsNullOrEmpty(pluginRecord?.Desc1))
|
||||
{
|
||||
_logger.LogWarning("CreateJobForLayer: Layer {LayerId} has no Plugin configured", layerId);
|
||||
return BadRequest("Layer has no Plugin configured");
|
||||
}
|
||||
|
||||
// Get priority and max retries
|
||||
var priorityRecord = layer.Records?.FirstOrDefault(x => x.Code == "Priority");
|
||||
var maxRetriesRecord = layer.Records?.FirstOrDefault(x => x.Code == "MaxRetries");
|
||||
|
||||
var priority = int.TryParse(priorityRecord?.Desc1, out var p) ? p : 0;
|
||||
var maxRetries = int.TryParse(maxRetriesRecord?.Desc1, out var m) ? m : 3;
|
||||
|
||||
var jobType = typeRecord.Desc1 == "ImportWorker" ? JobType.Import : JobType.Process;
|
||||
|
||||
// Check if there's already a pending/running job for this layer
|
||||
var existingJob = await _db.QueueJobs
|
||||
.Where(j => j.LayerId == layer.Id &&
|
||||
(j.Status == JobStatus.Pending || j.Status == JobStatus.Running))
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
if (existingJob != null)
|
||||
{
|
||||
_logger.LogInformation("CreateJobForLayer: Job already exists for layer {LayerId}, returning existing job", layerId);
|
||||
return Ok(new
|
||||
{
|
||||
success = true,
|
||||
jobId = existingJob.Id,
|
||||
message = "Job already exists for this layer",
|
||||
existing = true
|
||||
});
|
||||
}
|
||||
|
||||
// Create the job
|
||||
var job = new QueueJob
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
LayerId = layer.Id,
|
||||
LayerName = layer.Name ?? "Unknown",
|
||||
PluginName = pluginRecord.Desc1,
|
||||
JobType = jobType,
|
||||
Priority = priority,
|
||||
MaxRetries = maxRetries,
|
||||
Status = JobStatus.Pending,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
CreatedAtUtc = DateTime.UtcNow,
|
||||
ModifiedAtUtc = DateTime.UtcNow,
|
||||
CreatedById = Guid.Empty,
|
||||
ModifiedById = Guid.Empty
|
||||
};
|
||||
|
||||
_db.QueueJobs.Add(job);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
_logger.LogInformation("CreateJobForLayer: Created job {JobId} for layer {LayerName} ({LayerId})",
|
||||
job.Id, layer.Name, layerId);
|
||||
|
||||
return Ok(new
|
||||
{
|
||||
success = true,
|
||||
jobId = job.Id,
|
||||
message = "Job created successfully",
|
||||
existing = false
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CreateJobForLayer: Error creating job for layer {LayerId}", layerId);
|
||||
return BadRequest(ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,9 +39,11 @@
|
||||
|
||||
<Target Name="CopyPlugins" AfterTargets="Build">
|
||||
<MSBuild Projects="../DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj" Properties="Configuration=$(Configuration);TargetFramework=$(TargetFramework)" />
|
||||
<MSBuild Projects="../DiunaBI.Plugins.PedrolloPL/DiunaBI.Plugins.PedrolloPL.csproj" Properties="Configuration=$(Configuration);TargetFramework=$(TargetFramework)" />
|
||||
|
||||
<ItemGroup>
|
||||
<PluginFiles Include="../DiunaBI.Plugins.Morska/bin/$(Configuration)/$(TargetFramework)/DiunaBI.Plugins.Morska.dll" />
|
||||
<PluginFiles Include="../DiunaBI.Plugins.PedrolloPL/bin/$(Configuration)/$(TargetFramework)/DiunaBI.Plugins.PedrolloPL.dll" />
|
||||
</ItemGroup>
|
||||
<MakeDir Directories="$(OutputPath)Plugins" />
|
||||
<Copy SourceFiles="@(PluginFiles)" DestinationFolder="$(OutputPath)Plugins" />
|
||||
|
||||
@@ -49,7 +49,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("DataInbox");
|
||||
b.ToTable("DataInbox", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.Layer", b =>
|
||||
@@ -104,7 +104,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasIndex("ModifiedById");
|
||||
|
||||
b.ToTable("Layers");
|
||||
b.ToTable("Layers", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.ProcessSource", b =>
|
||||
@@ -119,7 +119,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasIndex("SourceId");
|
||||
|
||||
b.ToTable("ProcessSources");
|
||||
b.ToTable("ProcessSources", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.QueueJob", b =>
|
||||
@@ -185,7 +185,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("QueueJobs");
|
||||
b.ToTable("QueueJobs", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.Record", b =>
|
||||
@@ -329,7 +329,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasIndex("ModifiedById");
|
||||
|
||||
b.ToTable("Records");
|
||||
b.ToTable("Records", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.RecordHistory", b =>
|
||||
@@ -378,7 +378,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasIndex("RecordId", "ChangedAt");
|
||||
|
||||
b.ToTable("RecordHistory");
|
||||
b.ToTable("RecordHistory", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.User", b =>
|
||||
@@ -402,7 +402,7 @@ namespace DiunaBI.Infrastructure.Migrations
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("Users");
|
||||
b.ToTable("Users", (string)null);
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Domain.Entities.Layer", b =>
|
||||
|
||||
@@ -142,8 +142,16 @@ public class JobWorkerService : BackgroundService
|
||||
else
|
||||
{
|
||||
job.Status = JobStatus.Retrying;
|
||||
_logger.LogInformation("JobWorker: Job {JobId} will retry (attempt {RetryCount}/{MaxRetries})",
|
||||
job.Id, job.RetryCount, job.MaxRetries);
|
||||
|
||||
// Exponential backoff: wait before retrying based on attempt number
|
||||
var backoffDelay = GetBackoffDelay(job.RetryCount);
|
||||
|
||||
_logger.LogInformation("JobWorker: Job {JobId} will retry in {Delay} (attempt {RetryCount}/{MaxRetries})",
|
||||
job.Id, backoffDelay, job.RetryCount, job.MaxRetries);
|
||||
|
||||
// Wait before marking as pending again
|
||||
await Task.Delay(backoffDelay, stoppingToken);
|
||||
job.Status = JobStatus.Pending;
|
||||
}
|
||||
}
|
||||
finally
|
||||
@@ -157,4 +165,14 @@ public class JobWorkerService : BackgroundService
|
||||
_logger.LogInformation("JobWorker: Stopping service...");
|
||||
await base.StopAsync(stoppingToken);
|
||||
}
|
||||
|
||||
private static TimeSpan GetBackoffDelay(int retryCount)
|
||||
{
|
||||
return retryCount switch
|
||||
{
|
||||
1 => TimeSpan.FromSeconds(30), // 1st retry: 30 seconds
|
||||
2 => TimeSpan.FromMinutes(2), // 2nd retry: 2 minutes
|
||||
_ => TimeSpan.FromMinutes(5) // 3rd+ retry: 5 minutes
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
382
DiunaBI.Plugins.PedrolloPL/Importers/PedrolloPLImportP2.cs
Normal file
382
DiunaBI.Plugins.PedrolloPL/Importers/PedrolloPLImportP2.cs
Normal file
@@ -0,0 +1,382 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Plugins;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.PedrolloPL.Importers;
|
||||
|
||||
public class PedrolloPLImportP2 : BaseDataImporter
|
||||
{
|
||||
public override string ImporterType => "PedrolloPL.Import.P2";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<PedrolloPLImportP2> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? DataInboxName { get; set; }
|
||||
private string? DataInboxSource { get; set; }
|
||||
private string? StartDate { get; set; }
|
||||
private string? EndDate { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cached deserialized data
|
||||
private List<List<object>>? _cachedRawData;
|
||||
private DataInbox? _cachedDataInbox;
|
||||
private Dictionary<string, string>? _regionCodeMap;
|
||||
|
||||
public PedrolloPLImportP2(
|
||||
AppDbContext db,
|
||||
ILogger<PedrolloPLImportP2> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Clear cache at start
|
||||
_cachedRawData = null;
|
||||
_cachedDataInbox = null;
|
||||
_regionCodeMap = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
ValidateConfiguration();
|
||||
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
// Find and deserialize DataInbox data
|
||||
FindAndDeserializeDataInbox();
|
||||
|
||||
// Load region code mapping from dictionary layer
|
||||
LoadRegionCodeMapping();
|
||||
|
||||
// Map data from DataInbox to Layer records
|
||||
var mappedRecords = MapDataToRecords();
|
||||
|
||||
// Create new Import layer
|
||||
var importLayer = CreateImportLayer(importWorker);
|
||||
|
||||
// Save records to database
|
||||
SaveRecordsToLayer(importLayer, mappedRecords);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName} - Created {RecordCount} records",
|
||||
ImporterType, importWorker.Name, mappedRecords.Count);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clear cache after import
|
||||
_cachedRawData = null;
|
||||
_cachedDataInbox = null;
|
||||
_regionCodeMap = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
DataInboxName = GetRecordValue(importWorker.Records, "DataInboxName");
|
||||
DataInboxSource = GetRecordValue(importWorker.Records, "DataInboxSource");
|
||||
StartDate = GetRecordValue(importWorker.Records, "StartDate");
|
||||
EndDate = GetRecordValue(importWorker.Records, "EndDate");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ImporterType}: Configuration loaded - DataInboxName: {DataInboxName}, Source: {Source}, Year: {Year}, Period: {StartDate} to {EndDate}, Enabled: {IsEnabled}",
|
||||
ImporterType, DataInboxName, DataInboxSource, ImportYear, StartDate, EndDate, IsEnabled);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(DataInboxName)) errors.Add("DataInboxName is required");
|
||||
if (string.IsNullOrEmpty(DataInboxSource)) errors.Add("DataInboxSource is required");
|
||||
if (string.IsNullOrEmpty(StartDate)) errors.Add("StartDate is required");
|
||||
if (string.IsNullOrEmpty(EndDate)) errors.Add("EndDate is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration validated successfully", ImporterType);
|
||||
}
|
||||
|
||||
private void FindAndDeserializeDataInbox()
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Searching for DataInbox with Name='{DataInboxName}' and Source='{DataInboxSource}'",
|
||||
ImporterType, DataInboxName, DataInboxSource);
|
||||
|
||||
// Find DataInbox by Name and Source, order by CreatedAt descending to get the latest
|
||||
var dataInbox = _db.DataInbox
|
||||
.Where(x => x.Name == DataInboxName && x.Source == DataInboxSource)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataInbox == null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"DataInbox not found with Name='{DataInboxName}' and Source='{DataInboxSource}'");
|
||||
}
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Found DataInbox - Id: {DataInboxId}, Name: {Name}, Source: {Source}, CreatedAt: {CreatedAt}",
|
||||
ImporterType, dataInbox.Id, dataInbox.Name, dataInbox.Source, dataInbox.CreatedAt);
|
||||
|
||||
// Deserialize the data
|
||||
try
|
||||
{
|
||||
var data = Convert.FromBase64String(dataInbox.Data);
|
||||
var jsonString = Encoding.UTF8.GetString(data);
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
|
||||
ImporterType, data.Length);
|
||||
|
||||
// Deserialize as array of arrays: [["<nieznany>", 1183.15, ...], ["DOLNOŚLĄSKIE", ...]]
|
||||
var rawData = JsonSerializer.Deserialize<List<List<object>>>(jsonString);
|
||||
if (rawData == null || rawData.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
|
||||
}
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully deserialized {RowCount} rows from DataInbox",
|
||||
ImporterType, rawData.Count);
|
||||
|
||||
// Log first few rows for debugging
|
||||
if (rawData.Count > 0)
|
||||
{
|
||||
var sampleSize = Math.Min(3, rawData.Count);
|
||||
_logger.LogDebug("{ImporterType}: Sample rows (first {SampleSize}):", ImporterType, sampleSize);
|
||||
for (int i = 0; i < sampleSize; i++)
|
||||
{
|
||||
var row = rawData[i];
|
||||
if (row.Count > 0)
|
||||
{
|
||||
var regionName = row[0]?.ToString() ?? "null";
|
||||
var valueCount = row.Count - 1;
|
||||
_logger.LogDebug(" [{Index}] Region: {Region}, Values: {ValueCount}",
|
||||
i, regionName, valueCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the deserialized data
|
||||
_cachedRawData = rawData;
|
||||
_cachedDataInbox = dataInbox;
|
||||
}
|
||||
catch (FormatException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxId}",
|
||||
ImporterType, dataInbox.Id);
|
||||
throw new InvalidOperationException($"Invalid base64 data in DataInbox: {dataInbox.Name}", e);
|
||||
}
|
||||
catch (JsonException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxId}",
|
||||
ImporterType, dataInbox.Id);
|
||||
throw new InvalidOperationException($"Invalid JSON data in DataInbox: {dataInbox.Name}", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadRegionCodeMapping()
|
||||
{
|
||||
const string dictionaryLayerName = "L1-D-P2-CODES";
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Loading region code mapping from dictionary layer '{DictionaryLayerName}'",
|
||||
ImporterType, dictionaryLayerName);
|
||||
|
||||
var dictionaryLayer = _db.Layers
|
||||
.Where(x => x.Name == dictionaryLayerName && x.Type == LayerType.Dictionary)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dictionaryLayer == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Dictionary layer '{dictionaryLayerName}' not found");
|
||||
}
|
||||
|
||||
// Load records for the dictionary layer
|
||||
var records = _db.Records
|
||||
.Where(x => x.LayerId == dictionaryLayer.Id)
|
||||
.ToList();
|
||||
|
||||
// Build mapping: Desc1 (region name) -> Code
|
||||
_regionCodeMap = records.ToDictionary(
|
||||
r => r.Desc1 ?? string.Empty,
|
||||
r => r.Code ?? string.Empty,
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Loaded {MappingCount} region code mappings",
|
||||
ImporterType, _regionCodeMap.Count);
|
||||
}
|
||||
|
||||
private List<Record> MapDataToRecords()
|
||||
{
|
||||
if (_cachedRawData == null)
|
||||
{
|
||||
throw new InvalidOperationException("Raw data not loaded. Call FindAndDeserializeDataInbox first.");
|
||||
}
|
||||
|
||||
if (_regionCodeMap == null)
|
||||
{
|
||||
throw new InvalidOperationException("Region code mapping not loaded. Call LoadRegionCodeMapping first.");
|
||||
}
|
||||
|
||||
var records = new List<Record>();
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Starting data mapping for {RowCount} rows",
|
||||
ImporterType, _cachedRawData.Count);
|
||||
|
||||
foreach (var row in _cachedRawData)
|
||||
{
|
||||
if (row.Count < 13)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Skipping row with insufficient data - expected 13 elements, got {Count}",
|
||||
ImporterType, row.Count);
|
||||
continue;
|
||||
}
|
||||
|
||||
// First element is region name
|
||||
var regionName = row[0]?.ToString();
|
||||
if (string.IsNullOrEmpty(regionName))
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Skipping row with empty region name", ImporterType);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find region code from dictionary
|
||||
if (!_regionCodeMap.TryGetValue(regionName, out var regionCode))
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Region code not found for '{RegionName}' - skipping",
|
||||
ImporterType, regionName);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create 12 records (one per month)
|
||||
for (int month = 1; month <= 12; month++)
|
||||
{
|
||||
var valueIndex = month; // row[1] = January, row[2] = February, etc.
|
||||
var valueObj = row[valueIndex];
|
||||
|
||||
// Convert value to double
|
||||
double? value = null;
|
||||
if (valueObj != null)
|
||||
{
|
||||
// Handle JsonElement if deserialized from JSON
|
||||
if (valueObj is JsonElement jsonElement)
|
||||
{
|
||||
if (jsonElement.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
value = jsonElement.GetDouble();
|
||||
}
|
||||
}
|
||||
else if (valueObj is double d)
|
||||
{
|
||||
value = d;
|
||||
}
|
||||
else if (double.TryParse(valueObj.ToString(), out var parsed))
|
||||
{
|
||||
value = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
// Create code: {regionCode}{month:00}
|
||||
var code = $"{regionCode}{month:00}";
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = code,
|
||||
Value1 = value,
|
||||
CreatedAt = now,
|
||||
ModifiedAt = now
|
||||
};
|
||||
|
||||
records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Mapped region '{RegionName}' (code: {RegionCode}) to 12 records",
|
||||
ImporterType, regionName, regionCode);
|
||||
}
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully mapped {RecordCount} records from {RowCount} rows",
|
||||
ImporterType, records.Count, _cachedRawData.Count);
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
private Layer CreateImportLayer(Layer importWorker)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
var importLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
IsCancelled = false,
|
||||
CreatedAt = now,
|
||||
ModifiedAt = now,
|
||||
CreatedById = importWorker.CreatedById,
|
||||
ModifiedById = importWorker.ModifiedById
|
||||
};
|
||||
|
||||
// Format: L{Number}-I-P2-{Year}-{Timestamp}
|
||||
importLayer.Name = $"L{importLayer.Number}-I-P2-{ImportYear}-{now:yyyyMMddHHmm}";
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Creating import layer '{LayerName}' (Number: {Number})",
|
||||
ImporterType, importLayer.Name, importLayer.Number);
|
||||
|
||||
_db.Layers.Add(importLayer);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Created import layer '{LayerName}' with Id: {LayerId}",
|
||||
ImporterType, importLayer.Name, importLayer.Id);
|
||||
|
||||
return importLayer;
|
||||
}
|
||||
|
||||
private void SaveRecordsToLayer(Layer importLayer, List<Record> records)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Saving {RecordCount} records to layer {LayerId}",
|
||||
ImporterType, records.Count, importLayer.Id);
|
||||
|
||||
// Set LayerId for all records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.LayerId = importLayer.Id;
|
||||
}
|
||||
|
||||
_db.Records.AddRange(records);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully saved {RecordCount} records to layer '{LayerName}'",
|
||||
ImporterType, records.Count, importLayer.Name);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
141
DiunaBI.UI.Shared/Components/JobListComponent.razor
Normal file
141
DiunaBI.UI.Shared/Components/JobListComponent.razor
Normal file
@@ -0,0 +1,141 @@
|
||||
@using MudBlazor.Internal
|
||||
@using DiunaBI.Domain.Entities
|
||||
<MudExpansionPanels Class="mb-4">
|
||||
<MudExpansionPanel Icon="@Icons.Material.Filled.FilterList"
|
||||
Text="Filters"
|
||||
Expanded="true">
|
||||
<MudGrid AlignItems="Center">
|
||||
<MudItem xs="12" sm="6" md="3">
|
||||
<MudSelect T="JobStatus?"
|
||||
@bind-Value="selectedStatus"
|
||||
Label="Status"
|
||||
Placeholder="All statuses"
|
||||
Clearable="true"
|
||||
OnClearButtonClick="OnStatusClear">
|
||||
@foreach (JobStatus status in Enum.GetValues(typeof(JobStatus)))
|
||||
{
|
||||
<MudSelectItem T="JobStatus?" Value="@status">@status.ToString()</MudSelectItem>
|
||||
}
|
||||
</MudSelect>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12" sm="6" md="3">
|
||||
<MudSelect T="JobType?"
|
||||
@bind-Value="selectedJobType"
|
||||
Label="Job Type"
|
||||
Placeholder="All types"
|
||||
Clearable="true"
|
||||
OnClearButtonClick="OnJobTypeClear">
|
||||
@foreach (JobType type in Enum.GetValues(typeof(JobType)))
|
||||
{
|
||||
<MudSelectItem T="JobType?" Value="@type">@type.ToString()</MudSelectItem>
|
||||
}
|
||||
</MudSelect>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12" sm="12" md="6" Class="d-flex justify-end align-center">
|
||||
<MudIconButton Icon="@Icons.Material.Filled.Refresh"
|
||||
OnClick="LoadJobs"
|
||||
Color="Color.Primary"
|
||||
Size="Size.Medium"
|
||||
Title="Refresh"/>
|
||||
<MudIconButton Icon="@Icons.Material.Filled.Clear"
|
||||
OnClick="ClearFilters"
|
||||
Color="Color.Default"
|
||||
Size="Size.Medium"
|
||||
Title="Clear filters"/>
|
||||
</MudItem>
|
||||
</MudGrid>
|
||||
</MudExpansionPanel>
|
||||
</MudExpansionPanels>
|
||||
|
||||
<MudDivider Class="my-4"></MudDivider>
|
||||
|
||||
<MudTable Items="jobs.Items"
|
||||
Dense="true"
|
||||
Hover="true"
|
||||
Loading="isLoading"
|
||||
LoadingProgressColor="Color.Primary"
|
||||
OnRowClick="@((TableRowClickEventArgs<QueueJob> args) => OnRowClick(args.Item))"
|
||||
T="QueueJob"
|
||||
Style="cursor: pointer;">
|
||||
<HeaderContent>
|
||||
<MudTh>Layer Name</MudTh>
|
||||
<MudTh>Plugin</MudTh>
|
||||
<MudTh>Type</MudTh>
|
||||
<MudTh>Status</MudTh>
|
||||
<MudTh>Priority</MudTh>
|
||||
<MudTh>Retry</MudTh>
|
||||
<MudTh>Created</MudTh>
|
||||
<MudTh>Last Attempt</MudTh>
|
||||
</HeaderContent>
|
||||
<RowTemplate Context="row">
|
||||
<MudTd DataLabel="Layer Name">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@row.LayerName
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Plugin">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@row.PluginName
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Type">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
<MudChip T="string" Size="Size.Small" Color="@GetJobTypeColor(row.JobType)">@row.JobType</MudChip>
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Status">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
<MudChip T="string" Size="Size.Small" Color="@GetStatusColor(row.Status)">@row.Status</MudChip>
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Priority">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@row.Priority
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Retry">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@row.RetryCount / @row.MaxRetries
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Created">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@row.CreatedAt.ToString("yyyy-MM-dd HH:mm")
|
||||
</div>
|
||||
</MudTd>
|
||||
<MudTd DataLabel="Last Attempt">
|
||||
<div @oncontextmenu="@(async (e) => await OnRowRightClick(e, row))" @oncontextmenu:preventDefault="true">
|
||||
@(row.LastAttemptAt?.ToString("yyyy-MM-dd HH:mm") ?? "-")
|
||||
</div>
|
||||
</MudTd>
|
||||
</RowTemplate>
|
||||
<NoRecordsContent>
|
||||
<MudText>No jobs to display</MudText>
|
||||
</NoRecordsContent>
|
||||
<LoadingContent>
|
||||
Loading...
|
||||
</LoadingContent>
|
||||
</MudTable>
|
||||
|
||||
@if (jobs.TotalCount > 0)
|
||||
{
|
||||
<MudGrid Class="mt-4" AlignItems="Center.Center">
|
||||
<MudItem xs="12" sm="6">
|
||||
<MudText Typo="Typo.body2">
|
||||
Results @((jobs.Page - 1) * jobs.PageSize + 1) - @Math.Min(jobs.Page * jobs.PageSize, jobs.TotalCount)
|
||||
of @jobs.TotalCount
|
||||
</MudText>
|
||||
</MudItem>
|
||||
<MudItem xs="12" sm="6" Class="d-flex justify-end">
|
||||
<MudPagination Count="jobs.TotalPages"
|
||||
Selected="jobs.Page"
|
||||
SelectedChanged="OnPageChanged"
|
||||
ShowFirstButton="true"
|
||||
ShowLastButton="true"
|
||||
Variant="Variant.Outlined"
|
||||
/>
|
||||
</MudItem>
|
||||
</MudGrid>
|
||||
}
|
||||
110
DiunaBI.UI.Shared/Components/JobListComponent.razor.cs
Normal file
110
DiunaBI.UI.Shared/Components/JobListComponent.razor.cs
Normal file
@@ -0,0 +1,110 @@
|
||||
using DiunaBI.UI.Shared.Services;
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using Microsoft.AspNetCore.Components.Web;
|
||||
using DiunaBI.Application.DTOModels.Common;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using MudBlazor;
|
||||
using Microsoft.JSInterop;
|
||||
|
||||
namespace DiunaBI.UI.Shared.Components;
|
||||
|
||||
public partial class JobListComponent : ComponentBase
|
||||
{
|
||||
[Inject] private JobService JobService { get; set; } = default!;
|
||||
[Inject] private ISnackbar Snackbar { get; set; } = default!;
|
||||
[Inject] private NavigationManager NavigationManager { get; set; } = default!;
|
||||
[Inject] private IJSRuntime JSRuntime { get; set; } = default!;
|
||||
|
||||
private PagedResult<QueueJob> jobs = new();
|
||||
private bool isLoading = false;
|
||||
private int currentPage = 1;
|
||||
private int pageSize = 50;
|
||||
private JobStatus? selectedStatus = null;
|
||||
private JobType? selectedJobType = null;
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await LoadJobs();
|
||||
}
|
||||
|
||||
private async Task LoadJobs()
|
||||
{
|
||||
isLoading = true;
|
||||
|
||||
try
|
||||
{
|
||||
jobs = await JobService.GetJobsAsync(currentPage, pageSize, selectedStatus, selectedJobType);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Loading jobs failed: {ex.Message}");
|
||||
Snackbar.Add("Failed to load jobs", Severity.Error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
isLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task OnPageChanged(int page)
|
||||
{
|
||||
currentPage = page;
|
||||
await LoadJobs();
|
||||
}
|
||||
|
||||
private async Task ClearFilters()
|
||||
{
|
||||
selectedStatus = null;
|
||||
selectedJobType = null;
|
||||
currentPage = 1;
|
||||
await LoadJobs();
|
||||
}
|
||||
|
||||
private async Task OnStatusClear()
|
||||
{
|
||||
selectedStatus = null;
|
||||
currentPage = 1;
|
||||
await LoadJobs();
|
||||
}
|
||||
|
||||
private async Task OnJobTypeClear()
|
||||
{
|
||||
selectedJobType = null;
|
||||
currentPage = 1;
|
||||
await LoadJobs();
|
||||
}
|
||||
|
||||
private void OnRowClick(QueueJob job)
|
||||
{
|
||||
NavigationManager.NavigateTo($"/jobs/{job.Id}");
|
||||
}
|
||||
|
||||
private async Task OnRowRightClick(MouseEventArgs e, QueueJob job)
|
||||
{
|
||||
var url = NavigationManager.ToAbsoluteUri($"/jobs/{job.Id}").ToString();
|
||||
await JSRuntime.InvokeVoidAsync("open", url, "_blank");
|
||||
}
|
||||
|
||||
private Color GetStatusColor(JobStatus status)
|
||||
{
|
||||
return status switch
|
||||
{
|
||||
JobStatus.Pending => Color.Default,
|
||||
JobStatus.Running => Color.Info,
|
||||
JobStatus.Completed => Color.Success,
|
||||
JobStatus.Failed => Color.Error,
|
||||
JobStatus.Retrying => Color.Warning,
|
||||
_ => Color.Default
|
||||
};
|
||||
}
|
||||
|
||||
private Color GetJobTypeColor(JobType jobType)
|
||||
{
|
||||
return jobType switch
|
||||
{
|
||||
JobType.Import => Color.Primary,
|
||||
JobType.Process => Color.Secondary,
|
||||
_ => Color.Default
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddScoped<AuthService>();
|
||||
services.AddScoped<LayerService>();
|
||||
services.AddScoped<DataInboxService>();
|
||||
services.AddScoped<JobService>();
|
||||
|
||||
// Filter state services (scoped to maintain state during user session)
|
||||
services.AddScoped<LayerFilterStateService>();
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
<MudNavLink Href="/dashboard" Icon="@Icons.Material.Filled.Dashboard">Dashboard</MudNavLink>
|
||||
<MudNavLink Href="/layers" Icon="@Icons.Material.Filled.Inventory">Layers</MudNavLink>
|
||||
<MudNavLink Href="/datainbox" Icon="@Icons.Material.Filled.Inbox">Data Inbox</MudNavLink>
|
||||
<MudNavLink Href="/jobs" Icon="@Icons.Material.Filled.WorkHistory">Jobs</MudNavLink>
|
||||
</MudNavMenu>
|
||||
</MudDrawer>
|
||||
|
||||
|
||||
245
DiunaBI.UI.Shared/Pages/JobDetailPage.razor
Normal file
245
DiunaBI.UI.Shared/Pages/JobDetailPage.razor
Normal file
@@ -0,0 +1,245 @@
|
||||
@page "/jobs/{id:guid}"
|
||||
@using DiunaBI.UI.Shared.Services
|
||||
@using DiunaBI.Domain.Entities
|
||||
@using MudBlazor
|
||||
@inject JobService JobService
|
||||
@inject NavigationManager NavigationManager
|
||||
@inject ISnackbar Snackbar
|
||||
|
||||
<MudCard>
|
||||
<MudCardHeader>
|
||||
<CardHeaderContent>
|
||||
<MudText Typo="Typo.h5">Job Details</MudText>
|
||||
</CardHeaderContent>
|
||||
<CardHeaderActions>
|
||||
@if (job != null && job.Status == JobStatus.Failed)
|
||||
{
|
||||
<MudButton Variant="Variant.Filled"
|
||||
Color="Color.Warning"
|
||||
OnClick="RetryJob"
|
||||
StartIcon="@Icons.Material.Filled.Refresh">
|
||||
Retry
|
||||
</MudButton>
|
||||
}
|
||||
@if (job != null && (job.Status == JobStatus.Pending || job.Status == JobStatus.Retrying))
|
||||
{
|
||||
<MudButton Variant="Variant.Filled"
|
||||
Color="Color.Error"
|
||||
OnClick="CancelJob"
|
||||
StartIcon="@Icons.Material.Filled.Cancel">
|
||||
Cancel
|
||||
</MudButton>
|
||||
}
|
||||
<MudButton Variant="Variant.Text"
|
||||
OnClick="GoBack"
|
||||
StartIcon="@Icons.Material.Filled.ArrowBack">
|
||||
Back to List
|
||||
</MudButton>
|
||||
</CardHeaderActions>
|
||||
</MudCardHeader>
|
||||
<MudCardContent>
|
||||
@if (isLoading)
|
||||
{
|
||||
<MudProgressLinear Color="Color.Primary" Indeterminate="true" />
|
||||
}
|
||||
else if (job == null)
|
||||
{
|
||||
<MudAlert Severity="Severity.Error">Job not found</MudAlert>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudGrid>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@job.LayerName"
|
||||
Label="Layer Name"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@job.PluginName"
|
||||
Label="Plugin Name"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12" md="4">
|
||||
<MudTextField Value="@job.JobType.ToString()"
|
||||
Label="Job Type"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="4">
|
||||
<MudTextField Value="@job.Status.ToString()"
|
||||
Label="Status"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"
|
||||
Adornment="Adornment.Start"
|
||||
AdornmentIcon="@GetStatusIcon(job.Status)"
|
||||
AdornmentColor="@GetStatusColor(job.Status)"/>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="4">
|
||||
<MudTextField Value="@job.Priority.ToString()"
|
||||
Label="Priority"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@job.CreatedAt.ToString("yyyy-MM-dd HH:mm:ss")"
|
||||
Label="Created At"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@(job.LastAttemptAt?.ToString("yyyy-MM-dd HH:mm:ss") ?? "-")"
|
||||
Label="Last Attempt At"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@(job.CompletedAt?.ToString("yyyy-MM-dd HH:mm:ss") ?? "-")"
|
||||
Label="Completed At"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
<MudItem xs="12" md="6">
|
||||
<MudTextField Value="@($"{job.RetryCount} / {job.MaxRetries}")"
|
||||
Label="Retry Count / Max Retries"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"/>
|
||||
</MudItem>
|
||||
|
||||
@if (!string.IsNullOrEmpty(job.LastError))
|
||||
{
|
||||
<MudItem xs="12">
|
||||
<MudTextField Value="@job.LastError"
|
||||
Label="Last Error"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
FullWidth="true"
|
||||
Lines="5"
|
||||
AdornmentIcon="@Icons.Material.Filled.Error"
|
||||
AdornmentColor="Color.Error"/>
|
||||
</MudItem>
|
||||
}
|
||||
|
||||
<MudItem xs="12">
|
||||
<MudDivider Class="my-4"/>
|
||||
</MudItem>
|
||||
|
||||
<MudItem xs="12">
|
||||
<MudButton Variant="Variant.Outlined"
|
||||
Color="Color.Primary"
|
||||
OnClick="@(() => NavigationManager.NavigateTo($"/layers/{job.LayerId}"))"
|
||||
StartIcon="@Icons.Material.Filled.Layers">
|
||||
View Layer Details
|
||||
</MudButton>
|
||||
</MudItem>
|
||||
</MudGrid>
|
||||
}
|
||||
</MudCardContent>
|
||||
</MudCard>
|
||||
|
||||
@code {
|
||||
[Parameter]
|
||||
public Guid Id { get; set; }
|
||||
|
||||
private QueueJob? job;
|
||||
private bool isLoading = true;
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await LoadJob();
|
||||
}
|
||||
|
||||
private async Task LoadJob()
|
||||
{
|
||||
isLoading = true;
|
||||
try
|
||||
{
|
||||
job = await JobService.GetJobByIdAsync(Id);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Loading job failed: {ex.Message}");
|
||||
Snackbar.Add("Failed to load job", Severity.Error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
isLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RetryJob()
|
||||
{
|
||||
if (job == null) return;
|
||||
|
||||
var success = await JobService.RetryJobAsync(job.Id);
|
||||
if (success)
|
||||
{
|
||||
Snackbar.Add("Job reset to Pending status", Severity.Success);
|
||||
await LoadJob();
|
||||
}
|
||||
else
|
||||
{
|
||||
Snackbar.Add("Failed to retry job", Severity.Error);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CancelJob()
|
||||
{
|
||||
if (job == null) return;
|
||||
|
||||
var success = await JobService.CancelJobAsync(job.Id);
|
||||
if (success)
|
||||
{
|
||||
Snackbar.Add("Job cancelled", Severity.Success);
|
||||
await LoadJob();
|
||||
}
|
||||
else
|
||||
{
|
||||
Snackbar.Add("Failed to cancel job", Severity.Error);
|
||||
}
|
||||
}
|
||||
|
||||
private void GoBack()
|
||||
{
|
||||
NavigationManager.NavigateTo("/jobs");
|
||||
}
|
||||
|
||||
private Color GetStatusColor(JobStatus status)
|
||||
{
|
||||
return status switch
|
||||
{
|
||||
JobStatus.Pending => Color.Default,
|
||||
JobStatus.Running => Color.Info,
|
||||
JobStatus.Completed => Color.Success,
|
||||
JobStatus.Failed => Color.Error,
|
||||
JobStatus.Retrying => Color.Warning,
|
||||
_ => Color.Default
|
||||
};
|
||||
}
|
||||
|
||||
private string GetStatusIcon(JobStatus status)
|
||||
{
|
||||
return status switch
|
||||
{
|
||||
JobStatus.Pending => Icons.Material.Filled.HourglassEmpty,
|
||||
JobStatus.Running => Icons.Material.Filled.PlayArrow,
|
||||
JobStatus.Completed => Icons.Material.Filled.CheckCircle,
|
||||
JobStatus.Failed => Icons.Material.Filled.Error,
|
||||
JobStatus.Retrying => Icons.Material.Filled.Refresh,
|
||||
_ => Icons.Material.Filled.Help
|
||||
};
|
||||
}
|
||||
}
|
||||
8
DiunaBI.UI.Shared/Pages/JobListPage.razor
Normal file
8
DiunaBI.UI.Shared/Pages/JobListPage.razor
Normal file
@@ -0,0 +1,8 @@
|
||||
@page "/jobs"
|
||||
@using DiunaBI.UI.Shared.Components
|
||||
|
||||
<PageTitle>Jobs</PageTitle>
|
||||
|
||||
<MudContainer MaxWidth="MaxWidth.ExtraExtraLarge">
|
||||
<JobListComponent />
|
||||
</MudContainer>
|
||||
@@ -3,7 +3,9 @@
|
||||
@using DiunaBI.Application.DTOModels
|
||||
@using MudBlazor
|
||||
@inject LayerService LayerService
|
||||
@inject JobService JobService
|
||||
@inject NavigationManager NavigationManager
|
||||
@inject ISnackbar Snackbar
|
||||
|
||||
<MudCard>
|
||||
<MudCardHeader>
|
||||
@@ -11,18 +13,24 @@
|
||||
<MudText Typo="Typo.h5">Layer Details</MudText>
|
||||
</CardHeaderContent>
|
||||
<CardHeaderActions>
|
||||
<!--
|
||||
<MudButton Variant="Variant.Text" OnClick="Export">Export</MudButton>
|
||||
@if (layer != null && layer.Type == LayerType.Administration)
|
||||
@if (layer != null && layer.Type == LayerType.Administration && IsWorkerLayer())
|
||||
{
|
||||
<MudButton Variant="Variant.Text" Href="@($"/layers/edit/{layer.Id}/duplicate")">Duplicate</MudButton>
|
||||
<MudButton Variant="Variant.Text" Href="@($"/layers/edit/{layer.Id}")">Edit</MudButton>
|
||||
}
|
||||
@if (layer != null && layer.Type == LayerType.Processed)
|
||||
<MudButton Variant="Variant.Filled"
|
||||
Color="Color.Primary"
|
||||
OnClick="RunNow"
|
||||
Disabled="isRunningJob"
|
||||
StartIcon="@Icons.Material.Filled.PlayArrow">
|
||||
@if (isRunningJob)
|
||||
{
|
||||
<MudButton Variant="Variant.Text" OnClick="ProcessLayer">Process Layer</MudButton>
|
||||
<MudProgressCircular Size="Size.Small" Indeterminate="true"/>
|
||||
<span style="margin-left: 8px;">Creating Job...</span>
|
||||
}
|
||||
else
|
||||
{
|
||||
<span>Run Now</span>
|
||||
}
|
||||
</MudButton>
|
||||
}
|
||||
-->
|
||||
<MudButton Variant="Variant.Text" OnClick="GoBack" StartIcon="@Icons.Material.Filled.ArrowBack">Back to List</MudButton>
|
||||
</CardHeaderActions>
|
||||
</MudCardHeader>
|
||||
|
||||
@@ -10,9 +10,6 @@ public partial class LayerDetailPage : ComponentBase
|
||||
[Parameter]
|
||||
public Guid Id { get; set; }
|
||||
|
||||
[Inject]
|
||||
private ISnackbar Snackbar { get; set; } = null!;
|
||||
|
||||
[Inject]
|
||||
private IDialogService DialogService { get; set; } = null!;
|
||||
|
||||
@@ -413,4 +410,54 @@ public partial class LayerDetailPage : ComponentBase
|
||||
{
|
||||
return userCache.TryGetValue(userId, out var username) ? username : string.Empty;
|
||||
}
|
||||
|
||||
// Run Now button methods
|
||||
private bool isRunningJob = false;
|
||||
|
||||
private bool IsWorkerLayer()
|
||||
{
|
||||
if (layer?.Records == null) return false;
|
||||
|
||||
var typeRecord = layer.Records.FirstOrDefault(x => x.Code == "Type");
|
||||
return typeRecord?.Desc1 == "ImportWorker" || typeRecord?.Desc1 == "ProcessWorker";
|
||||
}
|
||||
|
||||
private async Task RunNow()
|
||||
{
|
||||
if (layer == null) return;
|
||||
|
||||
isRunningJob = true;
|
||||
try
|
||||
{
|
||||
var result = await JobService.CreateJobForLayerAsync(layer.Id);
|
||||
|
||||
if (result != null && result.Success)
|
||||
{
|
||||
if (result.Existing)
|
||||
{
|
||||
Snackbar.Add($"Job already exists: {result.Message}", Severity.Info);
|
||||
}
|
||||
else
|
||||
{
|
||||
Snackbar.Add("Job created successfully!", Severity.Success);
|
||||
}
|
||||
|
||||
// Navigate to job detail page
|
||||
NavigationManager.NavigateTo($"/jobs/{result.JobId}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Snackbar.Add("Failed to create job", Severity.Error);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error creating job: {ex.Message}");
|
||||
Snackbar.Add($"Error creating job: {ex.Message}", Severity.Error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
isRunningJob = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
104
DiunaBI.UI.Shared/Services/JobService.cs
Normal file
104
DiunaBI.UI.Shared/Services/JobService.cs
Normal file
@@ -0,0 +1,104 @@
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using DiunaBI.Application.DTOModels.Common;
|
||||
using DiunaBI.Domain.Entities;
|
||||
|
||||
namespace DiunaBI.UI.Shared.Services;
|
||||
|
||||
public class JobService
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
public JobService(HttpClient httpClient)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
|
||||
private readonly JsonSerializerOptions _jsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
public async Task<PagedResult<QueueJob>> GetJobsAsync(int page = 1, int pageSize = 50, JobStatus? status = null, JobType? jobType = null, Guid? layerId = null)
|
||||
{
|
||||
var start = (page - 1) * pageSize;
|
||||
var query = $"Jobs?start={start}&limit={pageSize}";
|
||||
|
||||
if (status.HasValue)
|
||||
query += $"&status={(int)status.Value}";
|
||||
|
||||
if (jobType.HasValue)
|
||||
query += $"&jobType={(int)jobType.Value}";
|
||||
|
||||
if (layerId.HasValue)
|
||||
query += $"&layerId={layerId.Value}";
|
||||
|
||||
var response = await _httpClient.GetAsync(query);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync();
|
||||
var result = JsonSerializer.Deserialize<PagedResult<QueueJob>>(json, _jsonOptions);
|
||||
|
||||
return result ?? new PagedResult<QueueJob>();
|
||||
}
|
||||
|
||||
public async Task<QueueJob?> GetJobByIdAsync(Guid id)
|
||||
{
|
||||
var response = await _httpClient.GetAsync($"Jobs/{id}");
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return null;
|
||||
|
||||
return await response.Content.ReadFromJsonAsync<QueueJob>();
|
||||
}
|
||||
|
||||
public async Task<bool> RetryJobAsync(Guid id)
|
||||
{
|
||||
var response = await _httpClient.PostAsync($"Jobs/{id}/retry", null);
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
public async Task<bool> CancelJobAsync(Guid id)
|
||||
{
|
||||
var response = await _httpClient.DeleteAsync($"Jobs/{id}");
|
||||
return response.IsSuccessStatusCode;
|
||||
}
|
||||
|
||||
public async Task<JobStats?> GetStatsAsync()
|
||||
{
|
||||
var response = await _httpClient.GetAsync("Jobs/stats");
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return null;
|
||||
|
||||
return await response.Content.ReadFromJsonAsync<JobStats>();
|
||||
}
|
||||
|
||||
public async Task<CreateJobResult?> CreateJobForLayerAsync(Guid layerId)
|
||||
{
|
||||
var response = await _httpClient.PostAsync($"Jobs/create-for-layer/{layerId}", null);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return null;
|
||||
|
||||
return await response.Content.ReadFromJsonAsync<CreateJobResult>();
|
||||
}
|
||||
}
|
||||
|
||||
public class JobStats
|
||||
{
|
||||
public int Pending { get; set; }
|
||||
public int Running { get; set; }
|
||||
public int Completed { get; set; }
|
||||
public int Failed { get; set; }
|
||||
public int Retrying { get; set; }
|
||||
public int Total { get; set; }
|
||||
}
|
||||
|
||||
public class CreateJobResult
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public Guid JobId { get; set; }
|
||||
public string? Message { get; set; }
|
||||
public bool Existing { get; set; }
|
||||
}
|
||||
Reference in New Issue
Block a user