mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final fixes before tests

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-08 10:18:52 +02:00
parent 117a67ab58
commit ff41a71484
10 changed files with 374 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -676,147 +676,19 @@ public class LayersController : Controller
}
}
}
[HttpGet]
[Route("GetImportWorkers")]
[AllowAnonymous]
public async Task<IActionResult> GetImportWorkers()
{
try
{
var importWorkerLayers = await _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
&& x.Number == 5579
)
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("GetImportWorkers: Found {LayerCount} import worker layers to queue",
importWorkerLayers.Count);
int queuedCount = 0;
foreach (var importWorker in importWorkerLayers)
{
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("GetImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
importWorker.Name, importWorker.Id);
continue;
}
// Check if plugin exists
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, importWorker.Name, importWorker.Id);
continue;
}
var job = new QueueJob
{
LayerId = importWorker.Id,
LayerName = importWorker.Name ?? "Unknown",
PluginName = pluginName,
JobType = JobType.Import,
Priority = 0, // All imports have same priority
MaxRetries = 5,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("GetImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("GetImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new {
Message = $"Queued {queuedCount} import jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = importWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "GetImportWorkers: Error queuing import workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("CheckProcessors")]
[AllowAnonymous]
public IActionResult CheckProcessors()
{
// get list od all enabled processors and check if they has record 'Plugin'
var enabledProcessors = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
// get
_logger.LogInformation("CheckProcessors: Found {ProcessorCount} enabled processors", enabledProcessors.Count);
foreach (var processor in enabledProcessors)
{
var pluginRecord = processor.Records!.FirstOrDefault(x => x.Code == "Plugin");
if (pluginRecord == null)
{
_logger.LogWarning("CheckProcessors: No Plugin record found for processor {ProcessorName} ({ProcessorId}), skipping",
processor.Name, processor.Id);
continue;
}
var pluginName = pluginRecord.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("CheckProcessors: Empty Plugin name for processor {ProcessorName} ({ProcessorId}), skipping",
processor.Name, processor.Id);
continue;
}
var processorInstance = _pluginManager.GetProcessor(pluginName);
if (processorInstance == null)
{
_logger.LogWarning("CheckProcessors: Processor {PluginName} not found for {ProcessorName} ({ProcessorId}), skipping",
pluginName, processor.Name, processor.Id);
continue;
}
}
_logger.LogInformation("CheckProcessors: Completed checking processors");
return Ok();
}
[HttpGet]
[Route("AddPluginName")]
[AllowAnonymous]
public IActionResult AddPluginName()
{
/*
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == "T1-R3")
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
@@ -829,20 +701,21 @@ public class LayersController : Controller
Id = Guid.NewGuid(),
LayerId = importWorker.Id,
Code = "Plugin",
Desc1 = "Morska.Process.T1.R3",
Desc1 = "Morska.Import.D3",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
_db.Records.Add(record);
//_db.Records.Add(record);
}
_db.SaveChanges();
//_db.SaveChanges();
*/
return Ok();
}
private static void WriteToConsole(params string[] messages)
{
foreach (var message in messages)
@@ -936,4 +809,277 @@ public class LayersController : Controller
throw;
}
}
[HttpGet]
[Route("EnqueueImportWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var importWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}",
importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (importWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No import workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var importWorker in importWorkerLayers)
{
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
importWorker.Name, importWorker.Id);
continue;
}
// Check if plugin exists
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, importWorker.Name, importWorker.Id);
continue;
}
var job = new QueueJob
{
LayerId = importWorker.Id,
LayerName = importWorker.Name ?? "Unknown",
PluginName = pluginName,
JobType = JobType.Import,
Priority = 0, // All imports have same priority
MaxRetries = 5,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} import jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = importWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("EnqueueProcessWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var processWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}",
processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (processWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No process workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var processWorker in processWorkerLayers)
{
var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
processWorker.Name, processWorker.Id);
continue;
}
var processorType = processWorker.Records!.FirstOrDefault(x => x.Code == "ProcessorType")?.Desc1;
if (string.IsNullOrEmpty(processorType))
{
_logger.LogWarning("EnqueueProcessWorkers: No processor type found for layer {LayerName} ({LayerId}), skipping",
processWorker.Name, processWorker.Id);
continue;
}
// Check if processor exists
var processor = _pluginManager.GetProcessor(processorType);
if (processor == null)
{
_logger.LogWarning("EnqueueProcessWorkers: Processor {ProcessorType} not found for layer {LayerName} ({LayerId}), skipping",
processorType, processWorker.Name, processWorker.Id);
continue;
}
// Get priority from ProcessWorker record, default to 10 if not found
var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1;
var priority = 10; // Default priority
if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority))
{
priority = parsedPriority;
}
var job = new QueueJob
{
LayerId = processWorker.Id,
LayerName = processWorker.Name ?? "Unknown",
PluginName = processorType, // Use processorType as PluginName for process jobs
JobType = JobType.Process,
Priority = priority,
MaxRetries = 3,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with processor {ProcessorType}, priority {Priority}",
processWorker.Name, processWorker.Id, processorType, priority);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} process jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = processWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("RunQueueJobs/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> RunQueueJobs(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var queueSize = await _queueService.GetQueueCountAsync();
if (queueSize == 0)
{
return Ok(new
{
Message = "Queue is empty",
QueueSize = 0,
Status = "No jobs to process"
});
}
_logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize);
// PRZYWRÓĆ SINGLETON ACCESS:
var queueProcessor = HttpContext.RequestServices.GetRequiredService<JobQueueProcessor>();
queueProcessor.TriggerProcessing();
return Ok(new
{
Message = $"Queue processing triggered for {queueSize} jobs",
QueueSize = queueSize,
Status = "Processing started in background"
});
}
catch (Exception e)
{
_logger.LogError(e, "RunQueueJobs: Error triggering queue processing");
return BadRequest(e.ToString());
}
}
}