From be80c90657a0087b10291f648388b8e4823e8b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Wed, 20 Dec 2023 18:39:43 +0100 Subject: [PATCH] new processor: T3-MultiSourceCopySelectedCodes --- Utils/SQL_HELPERS.sql | 39 ++++- WebAPI/Controllers/LayersController.cs | 7 + WebAPI/dataProcessors/ProcessHelper.cs | 25 ++- ....MultiSourceCopySelectedCodes.processor.cs | 160 ++++++++++++++++++ 4 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs diff --git a/Utils/SQL_HELPERS.sql b/Utils/SQL_HELPERS.sql index c38072d..e7791d4 100644 --- a/Utils/SQL_HELPERS.sql +++ b/Utils/SQL_HELPERS.sql @@ -71,19 +71,34 @@ --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); -- -- --- ADD PROCESS LAYER +--ADD PROCESS LAYER --DECLARE @LayerId UNIQUEIDENTIFIER; --SET @LayerId = 'c433afde-5ca9-422a-9434-7985839dd414'; -- --INSERT INTO [diunabi-morska].[dbo].[Layers] --([Id], [Number], [Source], [Name], [CreatedAt], [ModifiedAt], [IsDeleted], --[CreatedById], [ModifiedById], [Type]) ---VALUES ((SELECT @LayerId), 197, 'Manual', 'L197-A-PW_K5-2023/11-202310301538', +--VALUES ((SELECT @LayerId), 490, 'Manual', 'L197-A-PW_AB-2023/11-202311201714', --GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2); -- --INSERT INTO [diunabi-morska].[dbo].[Records] --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) ---VALUES ((SELECT NEWID()), 'Source', 'L181-A-IW_K5-2023/11-202310291031', +--VALUES ((SELECT NEWID()), 'Source', 'K5', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Source', 'M7', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Source', 'PU', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Source', 'FK', --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); -- --INSERT INTO [diunabi-morska].[dbo].[Records] @@ -93,11 +108,27 @@ -- --INSERT INTO [diunabi-morska].[dbo].[Records] --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) ---VALUES ((SELECT NEWID()), 'ProcessType', 'T3-SingleSource', +--VALUES ((SELECT NEWID()), 'ProcessType', 'T3-MultiSourceSummarySelectedCodes', --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); -- --INSERT INTO [diunabi-morska].[dbo].[Records] --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) --VALUES ((SELECT NEWID()), 'IsEnabled', 'True', --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Year', '2023', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Month', '11', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId)); +-- +--INSERT INTO [diunabi-morska].[dbo].[Records] +--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId]) +--VALUES ((SELECT NEWID()), 'Codes', '501-503;505-505;581-595;601-603;605-605;681-695;701-703;705-705;781-795;801-803;805-805;881-895', +--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, '7dbb3e85-895d-4c23-b57d-aa8505a26d5f'); -- \ No newline at end of file diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index 104470d..9309ebf 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -233,6 +233,7 @@ namespace WebAPI.Controllers x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") && x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") && x.Records!.Any(x => x.Code == "ProcessType" && x.Desc1 == type) + && x.Number == 490 ) .OrderBy(x => x.CreatedAt) .ToList(); @@ -362,6 +363,12 @@ namespace WebAPI.Controllers processor.process(processWorker!); break; } + case "T3-MultiSourceCopySelectedCodes": + { + T3MultiSourceCopySelectedCodesProcessor processor = new T3MultiSourceCopySelectedCodesProcessor(db, googleSheetValues, this); + processor.process(processWorker!); + break; + } } logsController.AddEntry(new LogEntry { diff --git a/WebAPI/dataProcessors/ProcessHelper.cs b/WebAPI/dataProcessors/ProcessHelper.cs index 7381d52..5ccc22d 100644 --- a/WebAPI/dataProcessors/ProcessHelper.cs +++ b/WebAPI/dataProcessors/ProcessHelper.cs @@ -108,7 +108,6 @@ namespace DiunaBIWebAPI.dataProcessors break; } } - public static float? getValue(Record record, int number) { switch (number) @@ -181,6 +180,30 @@ namespace DiunaBIWebAPI.dataProcessors return null; } } + public static List parseCodes(string codes) + { + List codesList = new List(); + foreach (string code in codes.Split(';')) + { + string[] range = code.Split('-'); + if (range.Length == 1) + { + codesList.Add(int.Parse(range[0])); + } + else if (range.Length == 2) + { + for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++) + { + codesList.Add(i); + } + } + else + { + throw new Exception($"Invalid code range: {code}"); + } + } + return codesList; + } } } diff --git a/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs new file mode 100644 index 0000000..539c16c --- /dev/null +++ b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs @@ -0,0 +1,160 @@ +using DiunaBIWebAPI.dataProcessors; +using Google.Apis.Sheets.v4; +using Microsoft.EntityFrameworkCore; +using WebAPI.Controllers; +using WebAPI.Models; + +namespace WebAPI.dataProcessors +{ + public class T3MultiSourceCopySelectedCodesProcessor + { + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; + + public T3MultiSourceCopySelectedCodesProcessor( + AppDbContext _db, + SpreadsheetsResource.ValuesResource _googleSheetValues, + LayersController _controller) + { + db = _db; + googleSheetValues = _googleSheetValues; + controller = _controller; + } + + public void process(Layer processWorker) + { + int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + List? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (sources!.Count() == 0) + { + throw new Exception("Source record not found"); + } + string? codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; + if (codes == null) + { + throw new Exception("Codes record not found"); + } + // create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512] + List codesList = ProcessHelper.parseCodes(codes); + + Layer? processedLayer = db.Layers + .Where(x => x.ParentId == processWorker!.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + bool isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Source = "", + Type = LayerType.processed, + ParentId = processWorker!.Id, + Number = db.Layers.Count() + 1, + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + processedLayer.Sources = new List(); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + List dataSources = new List(); + foreach (Record source in sources!) + { + Layer? dataSource = db.Layers.Where(x => + x.Type == LayerType.processed && + !x.IsDeleted && + x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3") + ) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSource != null) + { + dataSources.Add(dataSource); + } + } + if (dataSources.Count == 0) + { + throw new Exception($"DataSources are empty"); + } + + + List newRecords = dataSources + .SelectMany(x => x.Records!) + .Where(x => codesList.Contains(int.Parse(x.Code!))) + .Select(x => + { + Record newRecord = new Record + { + Id = Guid.NewGuid(), + Code = x.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + for (var i = 1; i < 33; i++) { + ProcessHelper.setValue(newRecord, i, ProcessHelper.getValue(x, i)); + } + return newRecord; + }) + .ToList(); + + /* + foreach (Record baseRecord in dataSources.Last()?.Records!) + { + + List codeRecords = allRecords.Where(x => + x.Code!.Substring(1) == baseRecord.Code!.Substring(1)) + .ToList(); + List codeRecordsValidation = allRecords.Where(x => + x.Code!.Substring(1) == baseRecord.Code!.Substring(1)) + .ToList(); + Record processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"19{baseRecord.Code!.Substring(Math.Max(0, baseRecord.Code!.Length - 2))}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + Record validationRecord = new Record(); + for (var i = 1; i < 33; i++) + { + ProcessHelper.setValue(processedRecord, i, + codeRecords.Sum(x => ProcessHelper.getValue(x, i))); + + ProcessHelper.setValue(validationRecord, i, + codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i))); + + if ( + ProcessHelper.getValue(processedRecord,i) != + ProcessHelper.getValue(validationRecord, i)) + { + throw new Exception($"ValidationError: Code {baseRecord.Code!}, " + + $"Value{i} ({ProcessHelper.getValue(processedRecord, i)} | " + + $"{ProcessHelper.getValue(validationRecord, i)})"); + } + } + newRecords.Add(processedRecord); + } + */ + if (isNew) + { + db.Layers.Add(processedLayer); + } + else + { + db.Layers.Update(processedLayer); + } + controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + db.SaveChanges(); + } + } +}