using System.Globalization; using System.Text; using DiunaBIWebAPI.dataImporters; using Google.Apis.Sheets.v4; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using WebAPI.dataProcessors; using WebAPI.Exports; using WebAPI.Models; namespace WebAPI.Controllers; [ApiController] [Route("api/[controller]")] public class LayersController : Controller { private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues; private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly LogsController _logsController; public LayersController( AppDbContext db, GoogleSheetsHelper googleSheetsHelper, GoogleDriveHelper googleDriveHelper, IConfiguration configuration ) { _db = db; if (googleSheetsHelper.Service is not null) { _googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values; } _googleDriveHelper = googleDriveHelper; _configuration = configuration; _logsController = new LogsController(googleSheetsHelper, _configuration); } [HttpGet] public IActionResult GetAll(int start, int limit, string? name, LayerType? type) { try { var response = _db.Layers.Where(x => !x.IsDeleted); if (name != null) { response = response.Where(x => x.Name != null && x.Name.Contains(name)); } if (type != null) { response = response.Where(x => x.Type == type); } return Ok(response .OrderByDescending(x => x.Number) .Skip(start).Take(limit).ToList()); } catch (Exception e) { return BadRequest(e.ToString()); } } [HttpGet] [Route("{id:guid}")] public IActionResult Get(Guid id) { try { return Ok(_db.Layers .Include(x => x.CreatedBy) .Include(x => x.Records).First(x => x.Id == id && !x.IsDeleted)); } catch (Exception e) { return BadRequest(e.ToString()); } } [HttpGet] [Route("getForPowerBI/{apiKey}/{number:int}")] public IActionResult GetByNumber(string apiKey, int number) { if (apiKey != _configuration["apiKey"]) { _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - wrong apiKey ({number})", Type = LogEntryType.Warning, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); } try { if ( !Request.Headers.TryGetValue("Authorization", out var authHeader)) { _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - no authorization header ({number})", Type = LogEntryType.Warning, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - wrong auth header format ({number})", Type = LogEntryType.Warning, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); } var authValue = Encoding.UTF8.GetString(Convert.FromBase64String(credentialsArr[1])); var username = authValue.Split(':')[0]; var password = authValue.Split(':')[1]; if (username != _configuration["powerBI-user"] || password != _configuration["powerBI-pass"]) { _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - bad credentials ({number})", Type = LogEntryType.Warning, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); } _logsController.AddEntry(new LogEntry { Title = $"Sending data for layer {number}", Type = LogEntryType.Info, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Ok(_db.Layers .Include(x => x.CreatedBy) .Include(x => x.Records).First(x => x.Number == number && !x.IsDeleted)); } catch (Exception e) { _logsController.AddEntry(new LogEntry { Title = e.ToString(), Type = LogEntryType.Error, LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return BadRequest(e.ToString()); } } [HttpGet] [Route("exportToGoogleSheet/{id:guid}")] public IActionResult ExportToGoogleSheet(Guid id) { if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } var layer = _db.Layers .Include(x => x.Records!.OrderByDescending(y => y.Code)).First(x => x.Id == id && !x.IsDeleted); var export = new GoogleSheetExport(_googleDriveHelper, _googleSheetValues, _configuration); export.Export(layer); return Ok(true); } [HttpGet] [Route("AutoImport/{apiKey}")] [AllowAnonymous] public IActionResult AutoImport(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { return Unauthorized(); } if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") //&& x.Records!.Any(x => x.Code == "ImportType" && x.Desc1 == "FK2") ) .OrderBy(x => x.CreatedAt) .ToList(); try { if (!importWorkerLayers.Any()) { _logsController.AddEntry(new LogEntry { Title = "No Layers to import.", Type = LogEntryType.Info, LogType = LogType.Import, CreatedAt = DateTime.UtcNow }); return Ok(); } foreach (var importWorker in importWorkerLayers) { try { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; if (type == "FK2") { var importer = new MorskaFk2Importer(_db, _googleSheetValues, this); importer.Import(importWorker); _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Import, Message = "Success", CreatedAt = DateTime.UtcNow }); } else { var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; if (startDate == null) { throw new Exception("StartDate record nod found"); } var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; if (endDate == null) { throw new Exception("EndDate record nod found"); } var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = new MorskaImporter(_db, _googleSheetValues, this); importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Import, Message = "Success", CreatedAt = DateTime.UtcNow }); } else if (IsImportedLayerUpToDate(importWorker) == false) { var importer = new MorskaImporter(_db, _googleSheetValues, this); importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", Type = LogEntryType.Warning, LogType = LogType.Import, Message = "Success (reimported)", CreatedAt = DateTime.UtcNow }); } else { _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", Type = LogEntryType.Warning, LogType = LogType.Import, Message = "importLayer records are up of date. Not processed.", CreatedAt = DateTime.UtcNow }); } } } catch (Exception e) { _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", Type = LogEntryType.Error, LogType = LogType.Import, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); } } return Ok(); } catch (Exception e) { _logsController.AddEntry(new LogEntry { Title = "Process error", Type = LogEntryType.Error, LogType = LogType.Import, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); return BadRequest(e.ToString()); } } [HttpGet] [Route("AutoProcess/{apiKey}")] [AllowAnonymous] public IActionResult AutoProcess(string apiKey) { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { return Unauthorized(); } if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } string[] processTypes = { "T3-SingleSource", "T3-SourceYearSummary", "T3-MultiSourceSummary", // AA "T3-MultiSourceYearSummary", // AA/13 "T4-SingleSource", "T1-R1", "T4-R2" }; foreach (var type in processTypes) { try { _logsController.AddEntry(new LogEntry { Title = $"Processing: {type}", Type = LogEntryType.Info, LogType = LogType.Process, CreatedAt = DateTime.UtcNow }); var processWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type) ) .OrderBy(x => x.CreatedAt) .ToList(); if (!processWorkerLayers.Any()) { _logsController.AddEntry(new LogEntry { Title = "No Layers to process.", Type = LogEntryType.Info, LogType = LogType.Process, CreatedAt = DateTime.UtcNow }); } foreach (var processWorker in processWorkerLayers) { try { ProcessLayer(processWorker); } catch (Exception e) { _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Error, LogType = LogType.Process, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); } } } catch (Exception e) { _logsController.AddEntry(new LogEntry { Title = "Process error", Type = LogEntryType.Error, LogType = LogType.Process, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); } } return Ok(); } private void ProcessLayer(Layer processWorker) { if (_googleSheetValues == null) { throw new Exception("Google Sheets API not initialized"); } var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; if (year == null) { throw new Exception("Year record nod found"); } var processType = processWorker.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1; switch (processType) { case null: throw new Exception("ProcessType record not found"); case "T3-SourceYearSummary": { var processor = new T3SourceYearSummaryProcessor(_db, this); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); return; } case "T3-MultiSourceYearSummary": { var processor = new T3MultiSourceYearSummaryProcessor(_db, this, _logsController); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); return; } case "T3-MultiSourceCopySelectedCodesYearSummary": { var processor = new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); return; } case "T1-R1": { var processor = new T1R1Processor(_db, _googleSheetValues, this, _logsController); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); return; } case "T4-R2": { var processor = new T4R2Processor(_db, this, _logsController); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); return; } } var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; if (month == null) { throw new Exception("Month record not found"); } switch (processType!) { case "T3-SingleSource": var t3SingleSource = new T3SingleSourceProcessor(_db, this); { t3SingleSource.Process(processWorker); break; } case "T4-SingleSource": { var t4SingleSource = new T4SingleSourceProcessor(_db, this); t4SingleSource.Process(processWorker); break; } case "T3-MultiSourceSummary": { var t3MultiSourceSummary = new T3MultiSourceSummaryProcessor(_db, this, _logsController); t3MultiSourceSummary.Process(processWorker); break; } case "T3-MultiSourceCopySelectedCodes": { var t3MultiSourceCopySelectedCode = new T3MultiSourceCopySelectedCodesProcessor(_db, this); t3MultiSourceCopySelectedCode.Process(processWorker); break; } } _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", Type = LogEntryType.Info, LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); } internal void SaveRecords(Guid id, ICollection records, Guid currentUserId) { var toDelete = _db.Records.Where(x => x.LayerId == id).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); } foreach (var record in records) { record.CreatedById = currentUserId; record.CreatedAt = DateTime.UtcNow; record.ModifiedById = currentUserId; record.ModifiedAt = DateTime.UtcNow; record.LayerId = id; _db.Records.Add(record); } } private static void WriteToConsole(params string[] messages) { foreach (var message in messages) { Console.WriteLine($"DiunaLog: {message}"); } } private bool IsImportedLayerUpToDate(Layer importWorker) { if (_googleSheetValues is null) { throw new Exception("Google Sheets API not initialized"); } var newestLayer = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == importWorker.Id) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); if (newestLayer is null) { return true; // importWorker is not active yet, no check needed } var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; if (dataRange == null) { throw new Exception($"DataRange not found, {importWorker.Name}"); } var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var data = dataRangeResponse.Values; var isUpToDate = true; for (var i = 0; i < data[1].Count; i++) { if (data[0][i].ToString() == "") continue; var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { WriteToConsole("Code not found in DiunaBI", data[0][i].ToString()!); isUpToDate = false; continue; } if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) || double.Abs((double)(record.Value1-value)!) > 0.01) continue; WriteToConsole( $"Code: {data[0][i]}. DiunaBI: {record.Value1:N2}. GoogleSheet: {data[1][i]}"); isUpToDate = false; } foreach (var record in newestLayer.Records!) { if (data[0].Contains(record.Code)) { continue; } WriteToConsole($"Code not found in GoogleSheet: {record.Code}"); isUpToDate = false; } return isUpToDate; } }