using System.Globalization; using System.Text; using Google.Apis.Sheets.v4; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using DiunaBI.Application.DTOModels; using DiunaBI.Application.DTOModels.Common; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Services; namespace DiunaBI.API.Controllers; [ApiController] [Route("[controller]")] public class LayersController : Controller { private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues; private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; private readonly ILogger _logger; public LayersController( AppDbContext db, SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, ILogger logger ) { _db = db; _googleSheetValues = googleSheetValues; _googleDriveHelper = googleDriveHelper; _configuration = configuration; _pluginManager = pluginManager; _logger = logger; } [HttpGet] [Route("")] public IActionResult GetAll([FromQuery] int start, [FromQuery] int limit, [FromQuery] string? name, [FromQuery] Domain.Entities.LayerType? type) { try { var query = _db.Layers.Where(x => !x.IsDeleted); if (name != null) { query = query.Where(x => x.Name != null && x.Name.Contains(name)); } if (type.HasValue) { query = query.Where(x => x.Type == type.Value); } var totalCount = query.Count(); var items = query .OrderByDescending(x => x.Number) .Skip(start) .Take(limit) .AsNoTracking() .Select(x => new LayerDto { Id = x.Id, Number = x.Number, Name = x.Name, Type = (Application.DTOModels.LayerType)x.Type, CreatedAt = x.CreatedAt, ModifiedAt = x.ModifiedAt, CreatedById = x.CreatedById, ModifiedById = x.ModifiedById, IsDeleted = x.IsDeleted, IsCancelled = x.IsCancelled, ParentId = x.ParentId }) .ToList(); var pagedResult = new PagedResult { Items = items, TotalCount = totalCount, Page = (start / limit) + 1, PageSize = limit }; _logger.LogDebug("GetAll: Retrieved {Count} of {TotalCount} layers (page {Page}) with filter name={Name}, type={Type}", items.Count, totalCount, pagedResult.Page, name, type); return Ok(pagedResult); } catch (Exception e) { _logger.LogError(e, "GetAll: Error retrieving layers"); return BadRequest(e.ToString()); } } [HttpGet] [Route("{id:guid}")] public IActionResult Get(Guid id) { try { var layer = _db.Layers .Include(x => x.CreatedBy) .Include(x => x.ModifiedBy) .Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); _logger.LogDebug("Get: Retrieved layer {LayerId} {LayerName}", id, layer.Name); return Ok(layer); } catch (Exception e) { _logger.LogError(e, "Get: Error retrieving layer {LayerId}", id); return BadRequest(e.ToString()); } } [HttpGet] [Route("getConfiguration/{apiKey}/{number:int}")] [AllowAnonymous] public IActionResult GetConfigurationByNumber(string apiKey, int number) { if (apiKey != _configuration["apiKey"]) { _logger.LogWarning("Configuration: Unauthorized request - wrong apiKey for layer {LayerNumber}", number); return Unauthorized(); } try { if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { _logger.LogWarning("Configuration: Unauthorized request - no authorization header for layer {LayerNumber}", number); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { _logger.LogWarning("Configuration: Unauthorized request - wrong auth header format for layer {LayerNumber}", number); return Unauthorized(); } var authValue = Encoding.UTF8.GetString(Convert.FromBase64String(credentialsArr[1])); var username = authValue.Split(':')[0]; var password = authValue.Split(':')[1]; if (username != _configuration["apiUser"] || password != _configuration["apiPass"]) { _logger.LogWarning("Configuration: Unauthorized request - bad credentials for layer {LayerNumber}", number); return Unauthorized(); } var config = _db.Layers .Include(x => x.Records) .AsNoTracking() .First(x => x.Number == number && !x.IsDeleted); if (config is null) { _logger.LogWarning("Configuration: Layer {LayerNumber} not found", number); return BadRequest(); } var type = config.Records?.Where(x => x.Code == "Type").FirstOrDefault(); if (type is null || type.Desc1 != "ExternalConfiguration") { _logger.LogWarning("Configuration: Layer {LayerNumber} is not ExternalConfiguration type", number); return BadRequest(); } _logger.LogInformation("Configuration: Sending configuration for layer {LayerNumber}", number); return Ok(config); } catch (Exception e) { _logger.LogError(e, "Configuration: Error occurred while processing layer {LayerNumber}", number); return BadRequest(); } } [HttpGet] [Route("exportToGoogleSheet/{id:guid}")] public IActionResult ExportToGoogleSheet(Guid id) { if (_googleSheetValues is null) { _logger.LogError("Export: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } try { var layer = _db.Layers .Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); var export = _pluginManager.GetExporter("GoogleSheet"); if (export == null) { _logger.LogError("Export: GoogleSheet exporter not found for layer {LayerId}", id); throw new Exception("GoogleSheet exporter not found"); } _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); export.Export(layer); _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); return Ok(true); } catch (Exception e) { _logger.LogError(e, "Export: Failed to export layer {LayerId} to GoogleSheet", id); throw; } } [HttpGet] [Route("ProcessQueue/{apiKey}")] [AllowAnonymous] public IActionResult ProcessQueue(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { _logger.LogWarning("ProcessQueue: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } _logger.LogInformation("ProcessQueue: Starting queue processing"); // Queue processing implementation would go here return Ok(); } [HttpGet] [Route("AutoImport/{apiKey}/{nameFilter}")] [AllowAnonymous] public IActionResult AutoImport(string apiKey, string nameFilter) { if (apiKey != _configuration["apiKey"]) { _logger.LogWarning("AutoImport: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { _logger.LogError("AutoImport: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Name != null && x.Name.Contains(nameFilter) && x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True" ) ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", nameFilter, importWorkerLayers.Count); try { if (importWorkerLayers.Count == 0) { _logger.LogInformation("AutoImport: No layers to import"); return Ok(); } foreach (var importWorker in importWorkerLayers) { try { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; var plugin = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; if (plugin == null) { _logger.LogError("AutoImport: Plugin not found for layer {LayerName} ({LayerId}), skipping", importWorker.Name, importWorker.Id); continue; } _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", importWorker.Name, type, source); if (source == "DataInbox" && type == "Import-D3") { var d3Importer = _pluginManager.GetImporter(plugin); if (d3Importer == null) { throw new Exception("MorskaD3 importer not found"); } d3Importer.Import(importWorker); _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); continue; } switch (type) { case "D1": var d1Importer = _pluginManager.GetImporter(plugin); if (d1Importer == null) { throw new Exception("MorskaD1 importer not found"); } d1Importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; case "FK2": var fk2Importer = _pluginManager.GetImporter(plugin); if (fk2Importer == null) { throw new Exception("MorskaFK2 importer not found"); } fk2Importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; default: var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; if (startDate == null) { throw new Exception("StartDate record not found"); } var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; if (endDate == null) { throw new Exception("EndDate record not found"); } var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = _pluginManager.GetImporter(plugin); if (importer == null) { throw new Exception("MorskaImporter not found"); } importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else if (IsImportedLayerUpToDate(importWorker) == false) { var importer = _pluginManager.GetImporter(plugin); if (importer == null) { throw new Exception("MorskaImporter not found"); } importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else { _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", importWorker.Name, importWorker.Id); } break; } } catch (Exception e) { _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } _logger.LogInformation("AutoImport: Completed processing {LayerCount} layers", importWorkerLayers.Count); return Ok(); } catch (Exception e) { _logger.LogError(e, "AutoImport: Process error"); return BadRequest(e.ToString()); } } [HttpGet] [Route("AutoProcess/{apiKey}")] [AllowAnonymous] public IActionResult AutoProcess(string apiKey) { if (apiKey != _configuration["apiKey"]) { _logger.LogWarning("AutoImport: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { _logger.LogError("AutoProcess: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } string[] processTypes = [ "T3-SingleSource", "T3-SourceYearSummary", "T3-MultiSourceSummary", "T3-MultiSourceYearSummary", "T4-SingleSource", "T5-LastValues", "T1-R1", "T4-R2", "T1-R3", "D6" ]; _logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length); foreach (var type in processTypes) { try { var processWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type) ) .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToList(); _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", type, processWorkerLayers.Count); foreach (var processWorker in processWorkerLayers) { try { ProcessLayer(processWorker); _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } catch (Exception e) { _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } } } catch (Exception e) { _logger.LogError(e, "AutoProcess: Error processing type {ProcessType}", type); } } _logger.LogInformation("AutoProcess: Completed processing all process types"); return Ok(); } private void ProcessLayer(Layer processWorker) { if (_googleSheetValues == null) { throw new Exception("Google Sheets API not initialized"); } var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; if (year == null) { throw new Exception("Year record not found"); } var plugin = processWorker.Records?.SingleOrDefault(x => x.Code == "Plugin")?.Desc1; if (plugin == null) { throw new Exception("Plugin record not found"); } var processType = processWorker.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1; switch (processType) { case null: throw new Exception("ProcessType record not found"); case "T3-SourceYearSummary": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T3.SourceYearSummary processor not found"); } processor.Process(processWorker); return; } case "T3-MultiSourceYearSummary": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T3.MultiSourceYearSummary processor not found"); } processor.Process(processWorker); return; } case "T3-MultiSourceCopySelectedCodesYearSummary": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T3.MultiSourceCopySelectedCodesYearSummary processor not found"); } processor.Process(processWorker); return; } case "T1-R1": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T1.R1 processor not found"); } processor.Process(processWorker); return; } case "T4-R2": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T4.R2 processor not found"); } processor.Process(processWorker); return; } case "T1-R3": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("T1.R3 processor not found"); } processor.Process(processWorker); return; } case "D6": { var processor = _pluginManager.GetProcessor(plugin); if (processor == null) { throw new Exception("D6 processor not found"); } processor.Process(processWorker); return; } } var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; if (month == null) { throw new Exception("Month record not found"); } switch (processType!) { case "T3-SingleSource": { var t3SingleSource = _pluginManager.GetProcessor(plugin); if (t3SingleSource == null) { throw new Exception("T3.SingleSource processor not found"); } t3SingleSource.Process(processWorker); break; } case "T4-SingleSource": { var t4SingleSource = _pluginManager.GetProcessor(plugin); if (t4SingleSource == null) { throw new Exception("T4.SingleSource processor not found"); } t4SingleSource.Process(processWorker); break; } case "T5-LastValues": { var t5LastValues = _pluginManager.GetProcessor(plugin); if (t5LastValues == null) { throw new Exception("T5.LastValues processor not found"); } t5LastValues.Process(processWorker); break; } case "T3-MultiSourceSummary": { var t3MultiSourceSummary = _pluginManager.GetProcessor(plugin); if (t3MultiSourceSummary == null) { throw new Exception("T3.MultiSourceSummary processor not found"); } t3MultiSourceSummary.Process(processWorker); break; } case "T3-MultiSourceCopySelectedCodes": { var t3MultiSourceCopySelectedCode = _pluginManager.GetProcessor(plugin); if (t3MultiSourceCopySelectedCode == null) { throw new Exception("T3.MultiSourceCopySelectedCodes processor not found"); } t3MultiSourceCopySelectedCode.Process(processWorker); break; } } } [HttpGet] [Route("CheckProcessors")] [AllowAnonymous] private static void WriteToConsole(params string[] messages) { foreach (var message in messages) { Console.WriteLine($"DiunaLog: {message}"); } } private bool IsImportedLayerUpToDate(Layer importWorker) { if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } var newestLayer = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == importWorker.Id) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (newestLayer is null) { _logger.LogDebug("IsImportedLayerUpToDate: No child layers found for {LayerName}, treating as up to date", importWorker.Name); return true; // importWorker is not active yet, no check needed } var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; if (dataRange == null) { throw new Exception($"DataRange not found, {importWorker.Name}"); } try { var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var data = dataRangeResponse.Values; var isUpToDate = true; for (var i = 0; i < data[1].Count; i++) { if (data[0][i].ToString() == "") continue; var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", data[0][i].ToString(), importWorker.Name); isUpToDate = false; continue; } if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) || double.Abs((double)(record.Value1 - value)!) < 0.01) continue; isUpToDate = false; } foreach (var record in newestLayer.Records!) { if (data[0].Contains(record.Code)) { continue; } _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", record.Code, importWorker.Name); isUpToDate = false; } _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; } catch (Exception e) { _logger.LogError(e, "IsImportedLayerUpToDate: Error checking if layer {LayerName} is up to date", importWorker.Name); throw; } } }