From 1fc00ac419dd9cde3bf713118a73c99bc29cca87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 21 Dec 2024 21:35:27 +0100 Subject: [PATCH] R3 processor --- .idea/.idea.DiunaBI/.idea/sqldialects.xml | 2 + WebAPI/Controllers/LayersController.cs | 16 ++ .../DbSeed/CreateProcessWorker-T1-R3.sql | 35 ++++ WebAPI/Helpers/HttpRequests/AutoProcess.http | 1 + WebAPI/dataProcessors/ProcessHelper.cs | 39 +++- WebAPI/dataProcessors/t1.r3.processor.cs | 172 ++++++++++++++++++ 6 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql create mode 100644 WebAPI/dataProcessors/t1.r3.processor.cs diff --git a/.idea/.idea.DiunaBI/.idea/sqldialects.xml b/.idea/.idea.DiunaBI/.idea/sqldialects.xml index 12912e7..208ced1 100644 --- a/.idea/.idea.DiunaBI/.idea/sqldialects.xml +++ b/.idea/.idea.DiunaBI/.idea/sqldialects.xml @@ -1,6 +1,8 @@ + + \ No newline at end of file diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index 3f20e99..3e6e751 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -461,6 +461,7 @@ public class LayersController : Controller "T1-R1_OLD", "T1-R1", "T4-R2", + "T1-R3" ]; foreach (var type in processTypes) @@ -632,6 +633,21 @@ public class LayersController : Controller var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues); processor.Process(processWorker); + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } + case "T1-R3": + { + var processor = new T1R3Processor(_db, this, _logsController, _googleSheetValues); + processor.Process(processWorker); + _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", diff --git a/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql b/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql new file mode 100644 index 0000000..7f05a59 --- /dev/null +++ b/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql @@ -0,0 +1,35 @@ +-- T1-R3 +DECLARE @LayerId UNIQUEIDENTIFIER; +SET @LayerId = NEWID(); + +INSERT INTO [diunabi-morska].[dbo].[Layers] +([Id], [Number], [Name], [CreatedAt], [ModifiedAt], [IsDeleted], + [CreatedById], [ModifiedById], [Type]) +VALUES ((SELECT @LayerId), 3953, 'L3953-A-PW_R3-2024-202412211332', + GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Source', 'D1', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'IsEnabled', 'True', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Year', '2024', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'Type', 'ProcessWorker', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + +INSERT INTO [diunabi-morska].[dbo].[Records] +([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +VALUES ((SELECT NEWID()), 'ProcessType', 'T1-R3', + GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); + diff --git a/WebAPI/Helpers/HttpRequests/AutoProcess.http b/WebAPI/Helpers/HttpRequests/AutoProcess.http index e9dd29e..97419fc 100644 --- a/WebAPI/Helpers/HttpRequests/AutoProcess.http +++ b/WebAPI/Helpers/HttpRequests/AutoProcess.http @@ -1,2 +1,3 @@ ### GET http://localhost:5400/api/Layers/AutoProcess/10763478CB738D4ecb2h76g803478CB738D4e +Timeout: 500000 diff --git a/WebAPI/dataProcessors/ProcessHelper.cs b/WebAPI/dataProcessors/ProcessHelper.cs index 7bb4e89..f85a621 100644 --- a/WebAPI/dataProcessors/ProcessHelper.cs +++ b/WebAPI/dataProcessors/ProcessHelper.cs @@ -1,4 +1,5 @@ -using WebAPI.Models; +using System.Text.RegularExpressions; +using WebAPI.Models; namespace DiunaBIWebAPI.dataProcessors; @@ -172,4 +173,40 @@ public static class ProcessHelper } return codesList; } + public static string? ExtractMonthFromLayerName(string layerName) + { + string pattern = @"L\d+-P-\d{4}/(\d{2})-"; + var match = Regex.Match(layerName, pattern); + if (match.Success && match.Groups.Count > 1) + { + return match.Groups[1].Value; + } + return null; + } + + public static string GetSheetName(int month, int year) + { + if (month < 1 || month > 12) + { + throw new ArgumentOutOfRangeException(nameof(month), "Month must be between 1 and 12."); + } + + var polishMonths = new[] + { + "Styczen", + "Luty", + "Marzec", + "Kwiecien", + "Maj", + "Czerwiec", + "Lipiec", + "Sierpien", + "Wrzesien", + "Pazdziernik", + "Listopad", + "Grudzien" + }; + var monthName = polishMonths[month - 1]; + return $"{monthName}_{year}"; + } } \ No newline at end of file diff --git a/WebAPI/dataProcessors/t1.r3.processor.cs b/WebAPI/dataProcessors/t1.r3.processor.cs new file mode 100644 index 0000000..0ee6106 --- /dev/null +++ b/WebAPI/dataProcessors/t1.r3.processor.cs @@ -0,0 +1,172 @@ +using System.Globalization; +using System.Text.RegularExpressions; +using DiunaBIWebAPI.dataProcessors; +using Google; +using Google.Apis.Sheets.v4; +using Google.Apis.Sheets.v4.Data; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.EntityFrameworkCore; +using WebAPI.Controllers; +using WebAPI.Models; + +namespace WebAPI.dataProcessors; + +public class T1R3Processor( + AppDbContext db, + LayersController controller, + LogsController logsController, + SpreadsheetsResource.ValuesResource googleSheetValues) +{ + public void Process(Layer processWorker) + { + + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1; + if (source == null) + { + throw new Exception("Source record not found"); + } + + var processedLayer = db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + + var newRecords = new List(); + + string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$"; + var dataSources = db.Layers + .Where(x => !x.IsDeleted) + .Include(layer => layer.Records!) // Filter on the database side as much as possible + .AsEnumerable() // Switch to in-memory evaluation + .Where(x => Regex.IsMatch(x.Name!, pattern)) // Apply Regex filtering + .ToList(); + + foreach (var dataSource in dataSources) + { + var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); + if (month == null) + { + throw new Exception($"Month not found: {dataSource.Name}"); + } + + foreach (var record in dataSource.Records!) + { + if (record.Value1 == null) continue; + for (var i = 1; i < 33; i++) + { + if (ProcessHelper.GetValue(record, i) == null) continue; + + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"{record.Code}{month}{i:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = i==1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100, + Desc1 = record.Desc1 + }; + + newRecords.Add(newRecord); + } + } + } + + if (isNew) + { + db.Layers.Add(processedLayer); + } + else + { + db.Layers.Update(processedLayer); + } + + controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + db.SaveChanges(); + + UpdateReport(); + } + + private void UpdateReport() + { + const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; + + var r3 = db.Layers + .Where(x => x.Number == 3954) + .Include(x => x.Records) + .FirstOrDefault(); + + for (var i = 1; i <= 12; i++) + { + var sheetName = ProcessHelper.GetSheetName(i, 2024); + ValueRange? dataRangeResponse; + try + { + dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); + } + catch (GoogleApiException ex) + { + continue; // Sheet not exist + } + + if (dataRangeResponse == null) continue; // Sheet not exist + var data = dataRangeResponse.Values; + + var updateValueRange = new ValueRange + { + Values = new List>() + }; + + foreach (var row in data) + { + if (row.Count == 0) continue; + var code = row[0].ToString(); + + var updateRow = new List(); + + for (var j = 1; j < 12; j++) + { + var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}"); + if (codeRecord is { Value1: not null }) + { + updateRow.Add(codeRecord.Value1); + } + else + { + updateRow.Add(""); + } + } + + updateValueRange.Values.Add(updateRow); + } + + dataRangeResponse.Values = data; + var update = googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + } + } +} \ No newline at end of file