R3 processor

2024-12-21 21:35:27 +01:00
parent dc3d4c19c3
commit 1fc00ac419
6 changed files with 264 additions and 1 deletions
--- a/.idea/.idea.DiunaBI/.idea/sqldialects.xml
+++ b/.idea/.idea.DiunaBI/.idea/sqldialects.xml
@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql" dialect="TSQL" />
+    <file url="file://$PROJECT_DIR$/WebAPI/Helpers/DbSeed/CreateProcessWorker-T3MultiSourceSummary-AA.sql" dialect="TSQL" />
    <file url="file://$PROJECT_DIR$/WebAPI/Helpers/DbSeed/CreateProcessWorker-T5LastValue.sql" dialect="TSQL" />
  </component>
 </project>
--- a/WebAPI/Controllers/LayersController.cs
+++ b/WebAPI/Controllers/LayersController.cs
@@ -461,6 +461,7 @@ public class LayersController : Controller
            "T1-R1_OLD",
            "T1-R1",
            "T4-R2",
+            "T1-R3"
        ];

        foreach (var type in processTypes)
@@ -632,6 +633,21 @@ public class LayersController : Controller
                var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues);
                processor.Process(processWorker);

+                _logsController.AddEntry(new LogEntry
+                {
+                    Title = $"{processWorker.Name}, {processWorker.Id}",
+                    Type = LogEntryType.Info,
+                    LogType = LogType.Process,
+                    Message = "Success",
+                    CreatedAt = DateTime.UtcNow
+                });
+                return;
+            }
+            case "T1-R3":
+            {
+                var processor = new T1R3Processor(_db, this, _logsController, _googleSheetValues);
+                processor.Process(processWorker);
+
                _logsController.AddEntry(new LogEntry
                {
                    Title = $"{processWorker.Name}, {processWorker.Id}",
--- a/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql
+++ b/WebAPI/Helpers/DbSeed/CreateProcessWorker-T1-R3.sql
@@ -0,0 +1,35 @@
+-- T1-R3
+DECLARE @LayerId UNIQUEIDENTIFIER;
+SET @LayerId = NEWID();
+
+INSERT INTO [diunabi-morska].[dbo].[Layers]
+([Id], [Number], [Name], [CreatedAt], [ModifiedAt], [IsDeleted],
+ [CreatedById], [ModifiedById], [Type])
+VALUES ((SELECT @LayerId), 3953, 'L3953-A-PW_R3-2024-202412211332',
+        GETDATE(), GETDATE(), 0, '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 2);
+
+INSERT INTO [diunabi-morska].[dbo].[Records]
+([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+VALUES ((SELECT NEWID()), 'Source', 'D1',
+        GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+
+INSERT INTO [diunabi-morska].[dbo].[Records]
+([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+VALUES ((SELECT NEWID()), 'IsEnabled', 'True',
+        GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+
+INSERT INTO [diunabi-morska].[dbo].[Records]
+([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+VALUES ((SELECT NEWID()), 'Year', '2024',
+        GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+
+INSERT INTO [diunabi-morska].[dbo].[Records]
+([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+VALUES ((SELECT NEWID()), 'Type', 'ProcessWorker',
+        GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+
+INSERT INTO [diunabi-morska].[dbo].[Records]
+([Id], [Code], [Desc1], [CreatedAt], [ModifiedAt], [CreatedById], [ModifiedById], [IsDeleted], [LayerId])
+VALUES ((SELECT NEWID()), 'ProcessType', 'T1-R3',
+        GETDATE(), GETDATE(), '117be4f0-b5d1-41a1-a962-39dc30cce368', '117be4f0-b5d1-41a1-a962-39dc30cce368', 0, (SELECT @LayerId));
+
--- a/WebAPI/Helpers/HttpRequests/AutoProcess.http
+++ b/WebAPI/Helpers/HttpRequests/AutoProcess.http
@@ -1,2 +1,3 @@
 ###
 GET http://localhost:5400/api/Layers/AutoProcess/10763478CB738D4ecb2h76g803478CB738D4e
+Timeout: 500000
--- a/WebAPI/dataProcessors/ProcessHelper.cs
+++ b/WebAPI/dataProcessors/ProcessHelper.cs
@@ -1,4 +1,5 @@
-using WebAPI.Models;
+using System.Text.RegularExpressions;
+using WebAPI.Models;

 namespace DiunaBIWebAPI.dataProcessors;

@@ -172,4 +173,40 @@ public static class ProcessHelper
        }
        return codesList;
    }
+    public static string? ExtractMonthFromLayerName(string layerName)
+    {
+        string pattern = @"L\d+-P-\d{4}/(\d{2})-";
+        var match = Regex.Match(layerName, pattern);
+        if (match.Success && match.Groups.Count > 1)
+        {
+            return match.Groups[1].Value;
+        }
+        return null;
+    }
+
+    public static string GetSheetName(int month, int year)
+    {
+        if (month < 1 || month > 12)
+        {
+            throw new ArgumentOutOfRangeException(nameof(month), "Month must be between 1 and 12.");
+        }
+
+        var polishMonths = new[]
+        {
+            "Styczen",    
+            "Luty",       
+            "Marzec",
+            "Kwiecien",
+            "Maj",
+            "Czerwiec",
+            "Lipiec",
+            "Sierpien",
+            "Wrzesien",
+            "Pazdziernik",
+            "Listopad",
+            "Grudzien"
+        };
+        var monthName = polishMonths[month - 1];
+        return $"{monthName}_{year}";
+    }
 }
--- a/WebAPI/dataProcessors/t1.r3.processor.cs
+++ b/WebAPI/dataProcessors/t1.r3.processor.cs
@@ -0,0 +1,172 @@
+using System.Globalization;
+using System.Text.RegularExpressions;
+using DiunaBIWebAPI.dataProcessors;
+using Google;
+using Google.Apis.Sheets.v4;
+using Google.Apis.Sheets.v4.Data;
+using Microsoft.AspNetCore.Http.HttpResults;
+using Microsoft.EntityFrameworkCore;
+using WebAPI.Controllers;
+using WebAPI.Models;
+
+namespace WebAPI.dataProcessors;
+
+public class T1R3Processor(
+    AppDbContext db,
+    LayersController controller,
+    LogsController logsController,
+    SpreadsheetsResource.ValuesResource googleSheetValues)
+{
+    public void Process(Layer processWorker)
+    {
+
+        var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
+        var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1;
+        if (source  == null)
+        {
+            throw new Exception("Source record not found");
+        }
+        
+        var processedLayer = db.Layers
+            .Where(x => x.ParentId == processWorker.Id
+                        && !x.IsDeleted)
+            .OrderByDescending(x => x.CreatedAt)
+            .FirstOrDefault();
+
+        var isNew = false;
+        if (processedLayer == null)
+        {
+            isNew = true;
+            processedLayer = new Layer
+            {
+                Id = Guid.NewGuid(),
+                Type = LayerType.Processed,
+                ParentId = processWorker.Id,
+                Number = db.Layers.Count() + 1
+            };
+            processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1";
+            processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.CreatedAt = DateTime.UtcNow;
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+        }
+        
+        processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+        processedLayer.ModifiedAt = DateTime.UtcNow;
+
+
+        var newRecords = new List<Record>();
+        
+        string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$";
+        var dataSources = db.Layers
+            .Where(x => !x.IsDeleted)
+            .Include(layer => layer.Records!) // Filter on the database side as much as possible
+            .AsEnumerable()           // Switch to in-memory evaluation
+            .Where(x => Regex.IsMatch(x.Name!, pattern)) // Apply Regex filtering
+            .ToList();
+
+        foreach (var dataSource in dataSources)
+        {
+            var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!);
+            if (month == null)
+            {
+                throw new Exception($"Month not found: {dataSource.Name}");
+            }
+
+            foreach (var record in dataSource.Records!)
+            {
+                if (record.Value1 == null) continue;
+                for (var i = 1; i < 33; i++)
+                {
+                    if (ProcessHelper.GetValue(record, i) == null) continue;
+                    
+                    var newRecord = new Record
+                    {
+                        Id = Guid.NewGuid(),
+                        Code = $"{record.Code}{month}{i:D2}",
+                        CreatedAt = DateTime.UtcNow,
+                        ModifiedAt = DateTime.UtcNow,
+                        Value1 = i==1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100,
+                        Desc1 = record.Desc1
+                    };
+                    
+                    newRecords.Add(newRecord);
+                }
+            }
+        }
+        
+        if (isNew)
+        {
+            db.Layers.Add(processedLayer);
+        }
+        else
+        {
+            db.Layers.Update(processedLayer);
+        }
+
+        controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
+        db.SaveChanges();
+        
+        UpdateReport();
+    }
+    
+        private void UpdateReport()
+    {
+        const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";
+
+        var r3 = db.Layers
+            .Where(x => x.Number == 3954)
+            .Include(x => x.Records)
+            .FirstOrDefault();
+
+        for (var i = 1; i <= 12; i++)
+        {
+            var sheetName = ProcessHelper.GetSheetName(i, 2024);
+            ValueRange? dataRangeResponse;
+            try
+            {
+                dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute();
+            }
+            catch (GoogleApiException ex)
+            {
+                continue; // Sheet not exist
+            }
+            
+            if (dataRangeResponse == null) continue; // Sheet not exist
+            var data = dataRangeResponse.Values;
+            
+            var updateValueRange = new ValueRange
+            {
+                Values = new List<IList<object>>()
+            };
+
+            foreach (var row in data)
+            {
+                if (row.Count == 0) continue;
+                var code = row[0].ToString();
+
+                var updateRow = new List<object>();
+                
+                for (var j = 1; j < 12; j++)
+                {
+                    var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}");
+                    if (codeRecord is { Value1: not null })
+                    {
+                        updateRow.Add(codeRecord.Value1);
+                    }
+                    else
+                    {
+                        updateRow.Add("");
+                    }
+                }
+                
+                updateValueRange.Values.Add(updateRow);
+            }
+            
+            dataRangeResponse.Values = data;
+            var update = googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200");
+            update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
+            update.Execute();
+        }
+    }
+}