AutoProcess Refactor. Logging improvement. MultiSourceSummaryProcessor

2023-11-06 22:52:07 +01:00
parent e7c413d9cf
commit 8c26698dc8
4 changed files with 297 additions and 106 deletions
--- a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
+++ b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
@@ -0,0 +1,116 @@
+using Google.Apis.Sheets.v4;
+using Microsoft.EntityFrameworkCore;
+using WebAPI.Controllers;
+using WebAPI.Models;
+
+namespace WebAPI.dataProcessors
+{
+    public class T3MultiSourceSummaryProcessor
+    {
+        private AppDbContext db;
+        private SpreadsheetsResource.ValuesResource googleSheetValues;
+        private LayersController controller;
+
+        public T3MultiSourceSummaryProcessor(
+            AppDbContext _db,
+            SpreadsheetsResource.ValuesResource _googleSheetValues,
+            LayersController _controller)
+        {
+            db = _db;
+            googleSheetValues = _googleSheetValues;
+            controller = _controller;
+        }
+
+        public void process(Layer processWorker)
+        {
+            int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
+            int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
+            List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
+            if (sources!.Count() == 0)
+            {
+                throw new Exception("Source record not found");
+            }
+
+            Layer? processedLayer = db.Layers
+                                    .Where(x => x.ParentId == processWorker!.Id
+                                                && !x.IsDeleted)
+                                    .OrderByDescending(x => x.CreatedAt)
+                                    .FirstOrDefault();
+
+            bool isNew = false;
+            if (processedLayer == null)
+            {
+                isNew = true;
+                processedLayer = new Layer
+                {
+                    Id = Guid.NewGuid(),
+                    Source = "",
+                    Type = LayerType.processed,
+                    ParentId = processWorker!.Id,
+                    Number = db.Layers.Count() + 1,
+                };
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3";
+                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.CreatedAt = DateTime.UtcNow;
+                processedLayer.ModifiedAt = DateTime.UtcNow;
+            }
+            processedLayer.Sources = new List<ProcessSource>();
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            List<Record> newRecords = new List<Record>();
+
+            List<Layer> dataSources = new List<Layer>();
+
+            foreach (Record source in sources!)
+            {
+                Layer? dataSource = db.Layers.Where(x =>
+                    x.Type == LayerType.processed &&
+                    !x.IsDeleted &&
+                    x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
+                    )
+                    .Include(x => x.Records)
+                    .FirstOrDefault();
+                if (dataSource != null)
+                {
+                    dataSources.Add(dataSource);
+                }
+            }
+
+            if (dataSources.Count == 0)
+            {
+                throw new Exception($"DataSources are empty");
+            }
+
+            List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
+            foreach (Record baseRecord in dataSources.Last()?.Records!)
+            {
+
+                List<Record> codeRecords = allRecords.Where(x =>
+                    x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
+                    .ToList();
+                Record processedRecord = new Record
+                {
+                    Id = Guid.NewGuid(),
+                    Code = $"9{baseRecord.Code!.Substring(1)}",
+                    CreatedAt = DateTime.UtcNow,
+                    ModifiedAt = DateTime.UtcNow,
+                    Value1 = codeRecords.Sum(x => x.Value32)
+                };
+                newRecords.Add(processedRecord);
+            }
+
+            if (isNew)
+            {
+                db.Layers.Add(processedLayer);
+            }
+            else
+            {
+                db.Layers.Update(processedLayer);
+            }
+            controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
+            db.SaveChanges();
+        }
+    }
+}
--- a/WebAPI/dataProcessors/t3.SingleSource.processor.cs
+++ b/WebAPI/dataProcessors/t3.SingleSource.processor.cs
@@ -24,55 +24,23 @@ namespace WebAPI.dataProcessors
            controller = _controller;
        }

-        public void process(Layer sourceImportWorker, Guid? parentId)
+        public void process(Layer processWorker)
        {
-            string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
-            if (sheetId == null)
-            {
-                throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
-            }
-            string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
-            if (sheetId == null)
-            {
-                throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
-            }
-            string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
-            if (importYearCell == null)
-            {
-                throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
-            }
-            string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
-            if (importMonthCell == null)
-            {
-                throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
-            }
-            string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
-            if (importNameCell == null)
-            {
-                throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
+            int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
+            int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
+            string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
+            if (source == null)
+            {
+                throw new Exception("Source record not found");
            }
-            var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
-            string? name = nameResponse.Values[0][0].ToString();
-            if (name == null)
-            {
-                throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
-            }
-            var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
-            int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
-            if (year == null)
-            {
-                throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
-            }
-            var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
-            string? month = monthResponse.Values[0][0].ToString();
-            if (month == null)
-            {
-                throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
+            Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == source);
+            if (sourceImportWorker == null)
+            {
+                throw new Exception("SourceImportWorkerL layer not found");
            }

-
            Layer? processedLayer = db.Layers
-                                    .Where(x => x.ParentId == parentId)
+                                    .Where(x => x.ParentId == processWorker!.Id)
                                    .OrderByDescending(x => x.CreatedAt)
                                    .FirstOrDefault();

@@ -85,10 +53,10 @@ namespace WebAPI.dataProcessors
                    Id = Guid.NewGuid(),
                    Source = "",
                    Type = LayerType.processed,
-                    ParentId = parentId,
+                    ParentId = processWorker!.Id,
                    Number = db.Layers.Count() + 1,
                };
-                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3";
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                processedLayer.CreatedAt = DateTime.UtcNow;
@@ -98,6 +66,7 @@ namespace WebAPI.dataProcessors
            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            processedLayer.ModifiedAt = DateTime.UtcNow;

+            
            List<Record> newRecords = new List<Record>();

            List<Layer> dataSources = db.Layers
@@ -110,16 +79,7 @@ namespace WebAPI.dataProcessors
            {
                throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
            }
-            /*
-            foreach (Layer source in dataSources)
-            {
-                response.Sources.Add(new ProcessSource
-                {
-                    LayerId = response.Id,
-                    SourceId = source.Id
-                });
-            }
-            */
+

            List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();

@@ -135,11 +95,11 @@ namespace WebAPI.dataProcessors
                    ModifiedAt = DateTime.UtcNow
                };

-                int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
+                int lastDayInMonth = DateTime.DaysInMonth(year, month);
                float previousValue = 0;
                //day 1
                float firstVal = codeRecords
-                    .Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
+                    .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
                    .OrderByDescending(x => x.CreatedAt)
                    .FirstOrDefault()?.Value1 ?? 0;
                setValue(processedRecord, 1, firstVal);
@@ -148,7 +108,7 @@ namespace WebAPI.dataProcessors
                for (int i=2; i<lastDayInMonth; i++)
                {
                    float? dayVal = codeRecords
-                        .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
+                        .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month)
                        .OrderByDescending(x => x.CreatedAt)
                        .FirstOrDefault()?.Value1;
                    if (dayVal == null)
@@ -164,7 +124,7 @@ namespace WebAPI.dataProcessors
                }
                //last day
                float? lastVal = codeRecords
-                    .Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
+                    .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth))
                    .OrderByDescending(x => x.CreatedAt)
                    .FirstOrDefault()?.Value1;

@@ -194,6 +154,7 @@ namespace WebAPI.dataProcessors
            }
            controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
            db.SaveChanges();
+            
        }
        private void setValue(Record record, int number, float? value)
        {
--- a/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
+++ b/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
@@ -7,29 +7,32 @@ namespace WebAPI.dataProcessors
 {
    public class T3SourceYearSummaryProcessor
    {
-        private AppDbContext db;
-        private SpreadsheetsResource.ValuesResource googleSheetValues;
-        private LayersController controller;
+        private readonly AppDbContext db;
+        private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
+        private readonly LayersController controller;
+        private readonly LogsController logsController;

        public T3SourceYearSummaryProcessor(
            AppDbContext _db,
            SpreadsheetsResource.ValuesResource _googleSheetValues,
-            LayersController _controller)
+            LayersController _controller,
+            LogsController _logsController
+            )
        {
            db = _db;
            googleSheetValues = _googleSheetValues;
            controller = _controller;
+            logsController = _logsController;
        }

        public void process(Layer processWorker)
        {
-            string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
-            int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0");
-            if (year == 0)
+            string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
+            string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
+            if (source == null)
            {
-                throw new Exception($"Year is empty, {processWorker!.Name}");
+                throw new Exception("Source record not found");
            }
-
            Layer? processedLayer = db.Layers
                                    .Where(x => x.ParentId == processWorker!.Id
                                                && !x.IsDeleted)
@@ -48,7 +51,7 @@ namespace WebAPI.dataProcessors
                    ParentId = processWorker!.Id,
                    Number = db.Layers.Count() + 1,
                };
-                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3";
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                processedLayer.CreatedAt = DateTime.UtcNow;
@@ -60,29 +63,29 @@ namespace WebAPI.dataProcessors

            List<Record> newRecords = new List<Record>();

-            List<Layer> sources = new List<Layer>();
+            List<Layer> dataSources = new List<Layer>();
            for (int i=1; i<13; i++)
            {
-                Layer? source = db.Layers.Where(x =>
+                Layer? dataSource = db.Layers.Where(x =>
                    x.Type == LayerType.processed
                    && !x.IsDeleted
-                    && x.Name.Contains($"{year}/{i}-{sourceName}-T3"))
+                    && x.Name.Contains($"{year}/{i}-{source}-T3"))
                    .Include(x => x.Records)
                    .FirstOrDefault();
-                if (source != null)
+                if (dataSource != null)
                {
-                    sources.Add(source);
+                    dataSources.Add(dataSource!);
                }
            }

-            if (sources.Count == 0)
+            if (dataSources.Count == 0)
            {
-                throw new Exception($"DataSources are empty, {processWorker!.Name}");
+                throw new Exception("DataSources are empty");
            }

-            List<Record> allRecords = sources.SelectMany(x => x.Records!).ToList();
+            List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();

-            foreach (Record baseRecord in sources.Last()?.Records!)
+            foreach (Record baseRecord in dataSources.Last()?.Records!)
            {
                List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
                Record processedRecord = new Record