new processor: T3-MultiSourceCopySelectedCodes

Utils/SQL_HELPERS.sql

@@ -71,19 +71,34 @@
 --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
 --
 --
--- ADD PROCESS LAYER
+--ADD PROCESS LAYER
 --DECLARE @LayerId UNIQUEIDENTIFIER;
 --SET @LayerId = 'c433afde-5ca9-422a-9434-7985839dd414';
 --
 --INSERT INTO [diunabi-morska].[dbo].[Layers] 
 --([Id], [Number], [Source], [Name], [CreatedAt], [ModifiedAt], [IsDeleted], 
 --[CreatedById], [ModifiedById], [Type])
---VALUES ((SELECT @LayerId), 197, 'Manual', 'L197-A-PW_K5-2023/11-202310301538',
+--VALUES ((SELECT @LayerId), 490, 'Manual', 'L197-A-PW_AB-2023/11-202311201714',
 --GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2);
 --
 --INSERT INTO [diunabi-morska].[dbo].[Records] 
 --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
---VALUES ((SELECT NEWID()), 'Source', 'L181-A-IW_K5-2023/11-202310291031',
+--VALUES ((SELECT NEWID()), 'Source', 'K5',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Source', 'M7',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Source', 'PU',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Source', 'FK',
 --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
 --
 --INSERT INTO [diunabi-morska].[dbo].[Records] 
@@ -93,11 +108,27 @@
 --
 --INSERT INTO [diunabi-morska].[dbo].[Records] 
 --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
---VALUES ((SELECT NEWID()), 'ProcessType', 'T3-SingleSource',
+--VALUES ((SELECT NEWID()), 'ProcessType', 'T3-MultiSourceSummarySelectedCodes',
 --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
 --
 --INSERT INTO [diunabi-morska].[dbo].[Records] 
 --([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
 --VALUES ((SELECT NEWID()), 'IsEnabled', 'True',
 --GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Year', '2023',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Month', '11',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+--
+--INSERT INTO [diunabi-morska].[dbo].[Records] 
+--([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+--VALUES ((SELECT NEWID()), 'Codes', '501-503;505-505;581-595;601-603;605-605;681-695;701-703;705-705;781-795;801-803;805-805;881-895',
+--GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, '7dbb3e85-895d-4c23-b57d-aa8505a26d5f');
 --

WebAPI/Controllers/LayersController.cs

@@ -233,6 +233,7 @@ namespace WebAPI.Controllers
                         x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
                         x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") &&
                         x.Records!.Any(x => x.Code == "ProcessType" && x.Desc1 == type)
+                        && x.Number == 490
                         )
                     .OrderBy(x => x.CreatedAt)
                     .ToList();
@@ -362,6 +363,12 @@ namespace WebAPI.Controllers
                                     processor.process(processWorker!);
                                     break;
                                 }
+                            case "T3-MultiSourceCopySelectedCodes":
+                                {
+                                    T3MultiSourceCopySelectedCodesProcessor processor = new T3MultiSourceCopySelectedCodesProcessor(db, googleSheetValues, this);
+                                    processor.process(processWorker!);
+                                    break;
+                                }
                         }
                         logsController.AddEntry(new LogEntry
                         {

WebAPI/dataProcessors/ProcessHelper.cs

@@ -108,7 +108,6 @@ namespace DiunaBIWebAPI.dataProcessors
                     break;
             }
         }
-
         public static float? getValue(Record record, int number)
         {
             switch (number)
@@ -181,6 +180,30 @@ namespace DiunaBIWebAPI.dataProcessors
                     return null;
             }
         }
+        public static List<int> parseCodes(string codes)
+        {
+            List<int> codesList = new List<int>();
+            foreach (string code in codes.Split(';'))
+            {
+                string[] range = code.Split('-');
+                if (range.Length == 1)
+                {
+                    codesList.Add(int.Parse(range[0]));
+                }
+                else if (range.Length == 2)
+                {
+                    for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++)
+                    {
+                        codesList.Add(i);
+                    }
+                }
+                else
+                {
+                    throw new Exception($"Invalid code range: {code}");
+                }
+            }
+            return codesList;
+        }
     }
 }
 

WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs

@@ -0,0 +1,160 @@
+using DiunaBIWebAPI.dataProcessors;
+using Google.Apis.Sheets.v4;
+using Microsoft.EntityFrameworkCore;
+using WebAPI.Controllers;
+using WebAPI.Models;
+
+namespace WebAPI.dataProcessors
+{
+    public class T3MultiSourceCopySelectedCodesProcessor
+    {
+        private readonly AppDbContext db;
+        private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
+        private readonly LayersController controller;
+
+        public T3MultiSourceCopySelectedCodesProcessor(
+            AppDbContext _db,
+            SpreadsheetsResource.ValuesResource _googleSheetValues,
+            LayersController _controller)
+        {
+            db = _db;
+            googleSheetValues = _googleSheetValues;
+            controller = _controller;
+        }
+
+        public void process(Layer processWorker)
+        {
+            int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
+            int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
+            List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
+            if (sources!.Count() == 0)
+            {
+                throw new Exception("Source record not found");
+            }
+            string? codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1;
+            if (codes == null)
+            {
+                throw new Exception("Codes record not found");
+            }
+            // create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512]
+            List<int> codesList = ProcessHelper.parseCodes(codes);
+
+            Layer? processedLayer = db.Layers
+                                    .Where(x => x.ParentId == processWorker!.Id
+                                                && !x.IsDeleted)
+                                    .OrderByDescending(x => x.CreatedAt)
+                                    .FirstOrDefault();
+
+            bool isNew = false;
+            if (processedLayer == null)
+            {
+                isNew = true;
+                processedLayer = new Layer
+                {
+                    Id = Guid.NewGuid(),
+                    Source = "",
+                    Type = LayerType.processed,
+                    ParentId = processWorker!.Id,
+                    Number = db.Layers.Count() + 1,
+                };
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3";
+                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.CreatedAt = DateTime.UtcNow;
+                processedLayer.ModifiedAt = DateTime.UtcNow;
+            }
+            processedLayer.Sources = new List<ProcessSource>();
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            List<Layer> dataSources = new List<Layer>();
+            foreach (Record source in sources!)
+            {
+                Layer? dataSource = db.Layers.Where(x =>
+                    x.Type == LayerType.processed &&
+                    !x.IsDeleted &&
+                    x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
+                    )
+                    .Include(x => x.Records)
+                    .FirstOrDefault();
+                if (dataSource != null)
+                {
+                    dataSources.Add(dataSource);
+                }
+            }
+            if (dataSources.Count == 0)
+            {
+                throw new Exception($"DataSources are empty");
+            }
+
+
+            List<Record> newRecords = dataSources
+                .SelectMany(x => x.Records!)
+                .Where(x => codesList.Contains(int.Parse(x.Code!)))
+                .Select(x =>
+                {
+                    Record newRecord = new Record
+                    {
+                        Id = Guid.NewGuid(),
+                        Code = x.Code,
+                        CreatedAt = DateTime.UtcNow,
+                        ModifiedAt = DateTime.UtcNow
+                    };
+                    for (var i = 1; i < 33; i++) {
+                        ProcessHelper.setValue(newRecord, i, ProcessHelper.getValue(x, i));
+                    }
+                    return newRecord;
+                })
+                .ToList();
+
+            /*
+                        foreach (Record baseRecord in dataSources.Last()?.Records!)
+                        {
+
+                            List<Record> codeRecords = allRecords.Where(x =>
+                                x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
+                                .ToList();
+                            List<Record> codeRecordsValidation = allRecords.Where(x =>
+                                x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
+                                .ToList();
+                            Record processedRecord = new Record
+                            {
+                                Id = Guid.NewGuid(),
+                                Code = $"19{baseRecord.Code!.Substring(Math.Max(0, baseRecord.Code!.Length - 2))}",
+                                CreatedAt = DateTime.UtcNow,
+                                ModifiedAt = DateTime.UtcNow
+                            };
+                            Record validationRecord = new Record();
+                            for (var i = 1; i < 33; i++)
+                            {
+                                ProcessHelper.setValue(processedRecord, i,
+                                    codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
+
+                                ProcessHelper.setValue(validationRecord, i,
+                                    codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i)));
+
+                                if (
+                                    ProcessHelper.getValue(processedRecord,i) !=
+                                        ProcessHelper.getValue(validationRecord, i))
+                                {
+                                    throw new Exception($"ValidationError: Code {baseRecord.Code!}, " +
+                                        $"Value{i} ({ProcessHelper.getValue(processedRecord, i)} | " +
+                                        $"{ProcessHelper.getValue(validationRecord, i)})");
+                                }
+                            }
+                            newRecords.Add(processedRecord);
+                        }
+            */
+            if (isNew)
+            {
+                db.Layers.Add(processedLayer);
+            }
+            else
+            {
+                db.Layers.Update(processedLayer);
+            }
+            controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
+            db.SaveChanges();
+        }
+    }
+}