using System.Globalization; using System.Text; using Google.Apis.Sheets.v4; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using DiunaBI.Core.Interfaces; namespace DiunaBI.WebAPI.Controllers; [ApiController] [Route("api/[controller]")] public class LayersController : Controller { private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues; private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; private readonly ILogger _logger; private readonly IJobQueueService _queueService; public LayersController( AppDbContext db, SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, ILogger logger, IJobQueueService queueService ) { _db = db; _googleSheetValues = googleSheetValues; _googleDriveHelper = googleDriveHelper; _configuration = configuration; _pluginManager = pluginManager; _logger = logger; _queueService = queueService; } [HttpGet] public IActionResult GetAll(int start, int limit, string? name, LayerType? type) { try { var response = _db.Layers.Where(x => !x.IsDeleted); if (name != null) { response = response.Where(x => x.Name != null && x.Name.Contains(name)); } if (type != null) { response = response.Where(x => x.Type == type); } var result = response .OrderByDescending(x => x.Number) .Skip(start).Take(limit).AsNoTracking().ToList(); _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", result.Count, name, type); return Ok(result); } catch (Exception e) { _logger.LogError(e, "GetAll: Error retrieving layers"); return BadRequest(e.ToString()); } } [HttpGet] [Route("{id:guid}")] public IActionResult Get(Guid id) { try { var layer = _db.Layers .Include(x => x.CreatedBy) .Include(x => x.ModifiedBy) .Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); _logger.LogDebug("Get: Retrieved layer {LayerId} {LayerName}", id, layer.Name); return Ok(layer); } catch (Exception e) { _logger.LogError(e, "Get: Error retrieving layer {LayerId}", id); return BadRequest(e.ToString()); } } [HttpGet] [Route("getForPowerBI/{apiKey}/{number:int}")] public IActionResult GetByNumber(string apiKey, int number) { if (apiKey != _configuration["apiKey"]) { _logger.LogWarning("PowerBI: Unauthorized request - wrong apiKey for layer {LayerNumber}", number); return Unauthorized(); } try { if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { _logger.LogWarning("PowerBI: Unauthorized request - no authorization header for layer {LayerNumber}", number); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { _logger.LogWarning("PowerBI: Unauthorized request - wrong auth header format for layer {LayerNumber}", number); return Unauthorized(); } var authValue = Encoding.UTF8.GetString(Convert.FromBase64String(credentialsArr[1])); var username = authValue.Split(':')[0]; var password = authValue.Split(':')[1]; if (username != _configuration["powerBI-user"] || password != _configuration["powerBI-pass"]) { _logger.LogWarning("PowerBI: Unauthorized request - bad credentials for layer {LayerNumber}", number); return Unauthorized(); } _logger.LogInformation("PowerBI: Sending data for layer {LayerNumber}", number); var layer = _db.Layers .Include(x => x.CreatedBy) .Include(x => x.Records).AsNoTracking().First(x => x.Number == number && !x.IsDeleted); return Ok(layer); } catch (Exception e) { _logger.LogError(e, "PowerBI: Error occurred while processing layer {LayerNumber}", number); return BadRequest(e.ToString()); } } [HttpGet] [Route("getConfiguration/{apiKey}/{number:int}")] public IActionResult GetConfigurationByNumber(string apiKey, int number) { if (apiKey != _configuration["apiKey"]) { _logger.LogWarning("Configuration: Unauthorized request - wrong apiKey for layer {LayerNumber}", number); return Unauthorized(); } try { if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { _logger.LogWarning("Configuration: Unauthorized request - no authorization header for layer {LayerNumber}", number); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { _logger.LogWarning("Configuration: Unauthorized request - wrong auth header format for layer {LayerNumber}", number); return Unauthorized(); } var authValue = Encoding.UTF8.GetString(Convert.FromBase64String(credentialsArr[1])); var username = authValue.Split(':')[0]; var password = authValue.Split(':')[1]; if (username != _configuration["morska-user"] || password != _configuration["morska-pass"]) { _logger.LogWarning("Configuration: Unauthorized request - bad credentials for layer {LayerNumber}", number); return Unauthorized(); } var config = _db.Layers .Include(x => x.Records) .AsNoTracking() .First(x => x.Number == number && !x.IsDeleted); if (config is null) { _logger.LogWarning("Configuration: Layer {LayerNumber} not found", number); return BadRequest(); } var type = config.Records?.Where(x => x.Code == "Type").FirstOrDefault(); if (type is null || type.Desc1 != "ExternalConfiguration") { _logger.LogWarning("Configuration: Layer {LayerNumber} is not ExternalConfiguration type", number); return BadRequest(); } _logger.LogInformation("Configuration: Sending configuration for layer {LayerNumber}", number); return Ok(config); } catch (Exception e) { _logger.LogError(e, "Configuration: Error occurred while processing layer {LayerNumber}", number); return BadRequest(); } } [HttpGet] [Route("exportToGoogleSheet/{id:guid}")] public IActionResult ExportToGoogleSheet(Guid id) { if (_googleSheetValues is null) { _logger.LogError("Export: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } try { var layer = _db.Layers .Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); var export = _pluginManager.GetExporter("GoogleSheet"); if (export == null) { _logger.LogError("Export: GoogleSheet exporter not found for layer {LayerId}", id); throw new Exception("GoogleSheet exporter not found"); } _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); export.Export(layer); _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); return Ok(true); } catch (Exception e) { _logger.LogError(e, "Export: Failed to export layer {LayerId} to GoogleSheet", id); throw; } } [HttpGet] [Route("AutoImportWithQueue/{apiKey}")] [AllowAnonymous] public IActionResult AutoImportWithQueue(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") ) .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToList(); if (importWorkerLayers.Count == 0) { _logger.LogInformation("AutoImportQueue: No layers to import"); return Ok(); } _logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count); foreach (var importWorker in importWorkerLayers) { try { // Queue job implementation would go here _logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } catch (Exception e) { _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } return Ok(); } [HttpGet] [Route("ProcessQueue/{apiKey}")] [AllowAnonymous] public IActionResult ProcessQueue(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("ProcessQueue: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } _logger.LogInformation("ProcessQueue: Starting queue processing"); // Queue processing implementation would go here return Ok(); } [HttpGet] [Route("AutoImport/{apiKey}/{nameFilter}")] [AllowAnonymous] public IActionResult AutoImport(string apiKey, string nameFilter) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("AutoImport: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { _logger.LogError("AutoImport: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Name != null && x.Name.Contains(nameFilter) && x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True" ) ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", nameFilter, importWorkerLayers.Count); try { if (importWorkerLayers.Count == 0) { _logger.LogInformation("AutoImport: No layers to import"); return Ok(); } foreach (var importWorker in importWorkerLayers) { try { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", importWorker.Name, type, source); if (source == "DataInbox" && type == "Import-D3") { var d3Importer = _pluginManager.GetImporter("MorskaD3"); if (d3Importer == null) { throw new Exception("MorskaD3 importer not found"); } d3Importer.Import(importWorker); _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); continue; } switch (type) { case "D1": var d1importer = _pluginManager.GetImporter("MorskaD1"); if (d1importer == null) { throw new Exception("MorskaD1 importer not found"); } d1importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; case "FK2": var fk2importer = _pluginManager.GetImporter("MorskaFK2"); if (fk2importer == null) { throw new Exception("MorskaFK2 importer not found"); } fk2importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; default: var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; if (startDate == null) { throw new Exception("StartDate record not found"); } var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; if (endDate == null) { throw new Exception("EndDate record not found"); } var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = _pluginManager.GetImporter("MorskaImporter"); if (importer == null) { throw new Exception("MorskaImporter not found"); } importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else if (IsImportedLayerUpToDate(importWorker) == false) { var importer = _pluginManager.GetImporter("MorskaImporter"); if (importer == null) { throw new Exception("MorskaImporter not found"); } importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else { _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", importWorker.Name, importWorker.Id); } break; } } catch (Exception e) { _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } _logger.LogInformation("AutoImport: Completed processing {LayerCount} layers", importWorkerLayers.Count); return Ok(); } catch (Exception e) { _logger.LogError(e, "AutoImport: Process error"); return BadRequest(e.ToString()); } } [HttpGet] [Route("AutoProcess/{apiKey}")] [AllowAnonymous] public IActionResult AutoProcess(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("AutoProcess: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { _logger.LogError("AutoProcess: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } string[] processTypes = [ "T3-SingleSource", "T3-SourceYearSummary", "T3-MultiSourceSummary", "T3-MultiSourceYearSummary", "T4-SingleSource", "T5-LastValues", "T1-R1", "T4-R2", "T1-R3" ]; _logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length); foreach (var type in processTypes) { try { var processWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type) ) .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToList(); _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", type, processWorkerLayers.Count); foreach (var processWorker in processWorkerLayers) { try { ProcessLayer(processWorker); _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } catch (Exception e) { _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } } } catch (Exception e) { _logger.LogError(e, "AutoProcess: Error processing type {ProcessType}", type); } } _logger.LogInformation("AutoProcess: Completed processing all process types"); return Ok(); } private void ProcessLayer(Layer processWorker) { if (_googleSheetValues == null) { throw new Exception("Google Sheets API not initialized"); } var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; if (year == null) { throw new Exception("Year record not found"); } var processType = processWorker.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1; switch (processType) { case null: throw new Exception("ProcessType record not found"); case "T3-SourceYearSummary": { var processor = _pluginManager.GetProcessor("T3.SourceYearSummary"); if (processor == null) { throw new Exception("T3.SourceYearSummary processor not found"); } processor.Process(processWorker); return; } case "T3-MultiSourceYearSummary": { var processor = _pluginManager.GetProcessor("T3.MultiSourceYearSummary"); if (processor == null) { throw new Exception("T3.MultiSourceYearSummary processor not found"); } processor.Process(processWorker); return; } case "T3-MultiSourceCopySelectedCodesYearSummary": { var processor = _pluginManager.GetProcessor("T3.MultiSourceCopySelectedCodesYearSummary"); if (processor == null) { throw new Exception("T3.MultiSourceCopySelectedCodesYearSummary processor not found"); } processor.Process(processWorker); return; } case "T1-R1": { var processor = _pluginManager.GetProcessor("T1.R1"); if (processor == null) { throw new Exception("T1.R1 processor not found"); } processor.Process(processWorker); return; } case "T4-R2": { var processor = _pluginManager.GetProcessor("T4.R2"); if (processor == null) { throw new Exception("T4.R2 processor not found"); } processor.Process(processWorker); return; } case "T1-R3": { var processor = _pluginManager.GetProcessor("T1.R3"); if (processor == null) { throw new Exception("T1.R3 processor not found"); } processor.Process(processWorker); return; } } var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; if (month == null) { throw new Exception("Month record not found"); } switch (processType!) { case "T3-SingleSource": { var t3SingleSource = _pluginManager.GetProcessor("T3.SingleSource"); if (t3SingleSource == null) { throw new Exception("T3.SingleSource processor not found"); } t3SingleSource.Process(processWorker); break; } case "T4-SingleSource": { var t4SingleSource = _pluginManager.GetProcessor("T4.SingleSource"); if (t4SingleSource == null) { throw new Exception("T4.SingleSource processor not found"); } t4SingleSource.Process(processWorker); break; } case "T5-LastValues": { var t5LastValues = _pluginManager.GetProcessor("T5.LastValues"); if (t5LastValues == null) { throw new Exception("T5.LastValues processor not found"); } t5LastValues.Process(processWorker); break; } case "T3-MultiSourceSummary": { var t3MultiSourceSummary = _pluginManager.GetProcessor("T3.MultiSourceSummary"); if (t3MultiSourceSummary == null) { throw new Exception("T3.MultiSourceSummary processor not found"); } t3MultiSourceSummary.Process(processWorker); break; } case "T3-MultiSourceCopySelectedCodes": { var t3MultiSourceCopySelectedCode = _pluginManager.GetProcessor("T3.MultiSourceCopySelectedCodes"); if (t3MultiSourceCopySelectedCode == null) { throw new Exception("T3.MultiSourceCopySelectedCodes processor not found"); } t3MultiSourceCopySelectedCode.Process(processWorker); break; } } } [HttpGet] [Route("CheckProcessors")] [AllowAnonymous] public IActionResult CheckProcessors() { // get /* var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); foreach (var importWorker in importWorkerLayers) { var record = new Record { Id = Guid.NewGuid(), LayerId = importWorker.Id, Code = "Plugin", Desc1 = "Morska.Import.D3", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") }; //_db.Records.Add(record); } //_db.SaveChanges(); */ return Ok(); } private static void WriteToConsole(params string[] messages) { foreach (var message in messages) { Console.WriteLine($"DiunaLog: {message}"); } } private bool IsImportedLayerUpToDate(Layer importWorker) { if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } var newestLayer = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == importWorker.Id) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (newestLayer is null) { _logger.LogDebug("IsImportedLayerUpToDate: No child layers found for {LayerName}, treating as up to date", importWorker.Name); return true; // importWorker is not active yet, no check needed } var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; if (dataRange == null) { throw new Exception($"DataRange not found, {importWorker.Name}"); } try { var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var data = dataRangeResponse.Values; var isUpToDate = true; for (var i = 0; i < data[1].Count; i++) { if (data[0][i].ToString() == "") continue; var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", data[0][i].ToString(), importWorker.Name); isUpToDate = false; continue; } if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) || double.Abs((double)(record.Value1 - value)!) < 0.01) continue; isUpToDate = false; } foreach (var record in newestLayer.Records!) { if (data[0].Contains(record.Code)) { continue; } _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", record.Code, importWorker.Name); isUpToDate = false; } _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; } catch (Exception e) { _logger.LogError(e, "IsImportedLayerUpToDate: Error checking if layer {LayerName} is up to date", importWorker.Name); throw; } } [HttpGet] [Route("EnqueueImportWorkers/{apiKey}")] [AllowAnonymous] public async Task EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } try { var query = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && !x.IsDeleted && !x.IsCancelled ); // If specific layerId is provided, filter to that layer only if (layerId.HasValue) { query = query.Where(x => x.Id == layerId.Value); } var importWorkerLayers = await query .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToListAsync(); _logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}", importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); if (importWorkerLayers.Count == 0) { return Ok(new { Message = "No import workers found to queue", QueuedJobs = 0, TotalQueueSize = await _queueService.GetQueueCountAsync(), SkippedLayers = 0 }); } int queuedCount = 0; foreach (var importWorker in importWorkerLayers) { var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; if (string.IsNullOrEmpty(pluginName)) { _logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", importWorker.Name, importWorker.Id); continue; } // Check if plugin exists var importer = _pluginManager.GetImporter(pluginName); if (importer == null) { _logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping", pluginName, importWorker.Name, importWorker.Id); continue; } var job = new QueueJob { LayerId = importWorker.Id, LayerName = importWorker.Name ?? "Unknown", PluginName = pluginName, JobType = JobType.Import, Priority = 0, // All imports have same priority MaxRetries = 5, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") }; await _queueService.EnqueueJobAsync(job); queuedCount++; _logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}", importWorker.Name, importWorker.Id, pluginName); } var totalQueueSize = await _queueService.GetQueueCountAsync(); _logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}", queuedCount, totalQueueSize); return Ok(new { Message = $"Queued {queuedCount} import jobs", QueuedJobs = queuedCount, TotalQueueSize = totalQueueSize, SkippedLayers = importWorkerLayers.Count - queuedCount }); } catch (Exception e) { _logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers"); return BadRequest(e.ToString()); } } [HttpGet] [Route("EnqueueProcessWorkers/{apiKey}")] [AllowAnonymous] public async Task EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } try { var query = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && !x.IsDeleted && !x.IsCancelled ); // If specific layerId is provided, filter to that layer only if (layerId.HasValue) { query = query.Where(x => x.Id == layerId.Value); } var processWorkerLayers = await query .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToListAsync(); _logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}", processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); if (processWorkerLayers.Count == 0) { return Ok(new { Message = "No process workers found to queue", QueuedJobs = 0, TotalQueueSize = await _queueService.GetQueueCountAsync(), SkippedLayers = 0 }); } int queuedCount = 0; foreach (var processWorker in processWorkerLayers) { var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; if (string.IsNullOrEmpty(pluginName)) { _logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", processWorker.Name, processWorker.Id); continue; } var processorType = processWorker.Records!.FirstOrDefault(x => x.Code == "ProcessorType")?.Desc1; if (string.IsNullOrEmpty(processorType)) { _logger.LogWarning("EnqueueProcessWorkers: No processor type found for layer {LayerName} ({LayerId}), skipping", processWorker.Name, processWorker.Id); continue; } // Check if processor exists var processor = _pluginManager.GetProcessor(processorType); if (processor == null) { _logger.LogWarning("EnqueueProcessWorkers: Processor {ProcessorType} not found for layer {LayerName} ({LayerId}), skipping", processorType, processWorker.Name, processWorker.Id); continue; } // Get priority from ProcessWorker record, default to 10 if not found var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1; var priority = 10; // Default priority if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority)) { priority = parsedPriority; } var job = new QueueJob { LayerId = processWorker.Id, LayerName = processWorker.Name ?? "Unknown", PluginName = processorType, // Use processorType as PluginName for process jobs JobType = JobType.Process, Priority = priority, MaxRetries = 3, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") }; await _queueService.EnqueueJobAsync(job); queuedCount++; _logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with processor {ProcessorType}, priority {Priority}", processWorker.Name, processWorker.Id, processorType, priority); } var totalQueueSize = await _queueService.GetQueueCountAsync(); _logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}", queuedCount, totalQueueSize); return Ok(new { Message = $"Queued {queuedCount} process jobs", QueuedJobs = queuedCount, TotalQueueSize = totalQueueSize, SkippedLayers = processWorkerLayers.Count - queuedCount }); } catch (Exception e) { _logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers"); return BadRequest(e.ToString()); } } [HttpGet] [Route("RunQueueJobs/{apiKey}")] [AllowAnonymous] public async Task RunQueueJobs(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } try { var queueSize = await _queueService.GetQueueCountAsync(); if (queueSize == 0) { return Ok(new { Message = "Queue is empty", QueueSize = 0, Status = "No jobs to process" }); } _logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize); // PRZYWRÓĆ SINGLETON ACCESS: var queueProcessor = HttpContext.RequestServices.GetRequiredService(); queueProcessor.TriggerProcessing(); return Ok(new { Message = $"Queue processing triggered for {queueSize} jobs", QueueSize = queueSize, Status = "Processing started in background" }); } catch (Exception e) { _logger.LogError(e, "RunQueueJobs: Error triggering queue processing"); return BadRequest(e.ToString()); } } }