From 8c26698dc888cf59aaaac9536d2704c54b24a35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Mon, 6 Nov 2023 22:52:07 +0100 Subject: [PATCH] AutoProcess Refactor. Logging improvement. MultiSourceSummaryProcessor --- WebAPI/Controllers/LayersController.cs | 165 +++++++++++++++--- .../t3.MultiSourceSummary.processor.cs | 116 ++++++++++++ .../t3.SingleSource.processor.cs | 81 +++------ .../t3.SourceYearSummary.processor.cs | 41 +++-- 4 files changed, 297 insertions(+), 106 deletions(-) create mode 100644 WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index b999321..7c43edd 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -186,20 +186,143 @@ namespace WebAPI.Controllers return Unauthorized(); } - List processWorkerLayers; - List layersToProcess; try { - processWorkerLayers = db.Layers + List processWorkerLayers = db.Layers .Include(x => x.Records) .Where(x => x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") && x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") - //&& x.Number == 215 + //&& x.Number == 262 ) + .OrderBy(x => x.CreatedAt) .ToList(); - layersToProcess = new List(); + if (processWorkerLayers.Count() == 0) + { + logsController.AddEntry(new LogEntry + { + Title = "No Layers to process.", + Type = LogEntryType.info, + LogType = LogType.process, + CreatedAt = DateTime.UtcNow, + }); + return Ok(); + } + + foreach (Layer processWorker in processWorkerLayers) + { + try + { + string name = processWorker.Name; + string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; + if (year == null) + { + throw new Exception("Year record nod found"); + } + + if (int.Parse(year!) < DateTime.UtcNow.Year) + { + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.warning, + LogType = LogType.process, + Message = "processLayer is out of date. Should be disabled. Not processed.", + CreatedAt = DateTime.UtcNow + }); + continue; + } + + string? processType = processWorker?.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1; + if (processType == null) + { + throw new Exception("ProcessType record not found"); + } + if (processType == "T3-SourceYearSummary") + { + if (int.Parse(year!) < DateTime.UtcNow.Year) + { + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.warning, + LogType = LogType.process, + Message = "processLayer is out of date. Should be disabled. Not processed", + CreatedAt = DateTime.UtcNow + }); + continue; + } + T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this, logsController); + processor.process(processWorker!); + + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.info, + LogType = LogType.process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + continue; + } + string? month = processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; + if (month == null) + { + throw new Exception("Month record not found"); + } + if (int.Parse(month!) < DateTime.UtcNow.Month) + { + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.warning, + LogType = LogType.process, + Message = "processLayer is out of date. Should be disabled.", + CreatedAt = DateTime.UtcNow + }); + } + switch (processType!) + { + case "T3-SingleSource": + { + T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this); + processor.process(processWorker!); + break; + } + case "T3-MultiSourceSummary": + { + T3MultiSourceSummaryProcessor processor = new T3MultiSourceSummaryProcessor(db, googleSheetValues, this); + processor.process(processWorker!); + break; + } + } + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.info, + LogType = LogType.process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + } catch (Exception e) + { + logsController.AddEntry(new LogEntry + { + Title = $"{processWorker!.Name}, {processWorker.Id}", + Type = LogEntryType.error, + LogType = LogType.process, + Message = e.ToString(), + CreatedAt = DateTime.UtcNow + }); + } + } + + return Ok(); + + + + foreach (Layer processWorker in processWorkerLayers) { @@ -216,8 +339,8 @@ namespace WebAPI.Controllers }); return BadRequest(); } - string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1; - if (sourceName == null) + List? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList(); + if (sources != null && sources.Count() == 0) { logsController.AddEntry(new LogEntry { @@ -234,15 +357,17 @@ namespace WebAPI.Controllers { if (processWorker != null) { - T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this); - processor.process(processWorker); + //T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this); + // processor.process(processWorker); } - } - else + } else if (processType == "T3-MultiSourceSummary") + { + var tst = 5; + } else { Layer sourceLayer = db.Layers.Include(x => x.Records) - .Single(x => x.Name == sourceName); + .Single(x => x.Name == sources!.First().Desc1); if (sourceLayer == null) { logsController.AddEntry(new LogEntry @@ -262,21 +387,7 @@ namespace WebAPI.Controllers var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { - switch (processType) - { - case "Copy": - CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this); - cp.process(sourceLayer, processWorker?.Id); - break; - case "Deaggregation": - DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this); - dp.process(sourceLayer, processWorker?.Id); - break; - case "T3-SingleSource": - T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this); - processor.process(sourceLayer, processWorker?.Id); - break; - } + logsController.AddEntry(new LogEntry { Title = $"Process Success, {sourceLayer.Name}", diff --git a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs new file mode 100644 index 0000000..845d233 --- /dev/null +++ b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs @@ -0,0 +1,116 @@ +using Google.Apis.Sheets.v4; +using Microsoft.EntityFrameworkCore; +using WebAPI.Controllers; +using WebAPI.Models; + +namespace WebAPI.dataProcessors +{ + public class T3MultiSourceSummaryProcessor + { + private AppDbContext db; + private SpreadsheetsResource.ValuesResource googleSheetValues; + private LayersController controller; + + public T3MultiSourceSummaryProcessor( + AppDbContext _db, + SpreadsheetsResource.ValuesResource _googleSheetValues, + LayersController _controller) + { + db = _db; + googleSheetValues = _googleSheetValues; + controller = _controller; + } + + public void process(Layer processWorker) + { + int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + List? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (sources!.Count() == 0) + { + throw new Exception("Source record not found"); + } + + Layer? processedLayer = db.Layers + .Where(x => x.ParentId == processWorker!.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + bool isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Source = "", + Type = LayerType.processed, + ParentId = processWorker!.Id, + Number = db.Layers.Count() + 1, + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + processedLayer.Sources = new List(); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + List newRecords = new List(); + + List dataSources = new List(); + + foreach (Record source in sources!) + { + Layer? dataSource = db.Layers.Where(x => + x.Type == LayerType.processed && + !x.IsDeleted && + x.Name.Contains($"{year}/{month}-{source.Desc1}-T3") + ) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSource != null) + { + dataSources.Add(dataSource); + } + } + + if (dataSources.Count == 0) + { + throw new Exception($"DataSources are empty"); + } + + List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + foreach (Record baseRecord in dataSources.Last()?.Records!) + { + + List codeRecords = allRecords.Where(x => + x.Code!.Substring(1) == baseRecord.Code!.Substring(1)) + .ToList(); + Record processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"9{baseRecord.Code!.Substring(1)}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = codeRecords.Sum(x => x.Value32) + }; + newRecords.Add(processedRecord); + } + + if (isNew) + { + db.Layers.Add(processedLayer); + } + else + { + db.Layers.Update(processedLayer); + } + controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + db.SaveChanges(); + } + } +} diff --git a/WebAPI/dataProcessors/t3.SingleSource.processor.cs b/WebAPI/dataProcessors/t3.SingleSource.processor.cs index ca1e892..607b1cc 100644 --- a/WebAPI/dataProcessors/t3.SingleSource.processor.cs +++ b/WebAPI/dataProcessors/t3.SingleSource.processor.cs @@ -24,55 +24,23 @@ namespace WebAPI.dataProcessors controller = _controller; } - public void process(Layer sourceImportWorker, Guid? parentId) + public void process(Layer processWorker) { - string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetId not found, {sourceImportWorker.Name}"); - } - string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}"); - } - string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; - if (importYearCell == null) - { - throw new Exception($"ImportYear not found, {sourceImportWorker.Name}"); - } - string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; - if (importMonthCell == null) - { - throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}"); - } - string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; - if (importNameCell == null) - { - throw new Exception($"ImportName not found, {sourceImportWorker.Name}"); + int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; + if (source == null) + { + throw new Exception("Source record not found"); } - var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); - string? name = nameResponse.Values[0][0].ToString(); - if (name == null) - { - throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}"); - } - var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); - int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? ""); - if (year == null) - { - throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}"); - } - var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute(); - string? month = monthResponse.Values[0][0].ToString(); - if (month == null) - { - throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}"); + Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == source); + if (sourceImportWorker == null) + { + throw new Exception("SourceImportWorkerL layer not found"); } - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == parentId) + .Where(x => x.ParentId == processWorker!.Id) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); @@ -85,10 +53,10 @@ namespace WebAPI.dataProcessors Id = Guid.NewGuid(), Source = "", Type = LayerType.processed, - ParentId = parentId, + ParentId = processWorker!.Id, Number = db.Layers.Count() + 1, }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3"; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3"; processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedAt = DateTime.UtcNow; @@ -98,6 +66,7 @@ namespace WebAPI.dataProcessors processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; + List newRecords = new List(); List dataSources = db.Layers @@ -110,16 +79,7 @@ namespace WebAPI.dataProcessors { throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); } - /* - foreach (Layer source in dataSources) - { - response.Sources.Add(new ProcessSource - { - LayerId = response.Id, - SourceId = source.Id - }); - } - */ + List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); @@ -135,11 +95,11 @@ namespace WebAPI.dataProcessors ModifiedAt = DateTime.UtcNow }; - int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? "")); + int lastDayInMonth = DateTime.DaysInMonth(year, month); float previousValue = 0; //day 1 float firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1)) + .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault()?.Value1 ?? 0; setValue(processedRecord, 1, firstVal); @@ -148,7 +108,7 @@ namespace WebAPI.dataProcessors for (int i=2; i x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? "")) + .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault()?.Value1; if (dayVal == null) @@ -164,7 +124,7 @@ namespace WebAPI.dataProcessors } //last day float? lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth)) + .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault()?.Value1; @@ -194,6 +154,7 @@ namespace WebAPI.dataProcessors } controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); db.SaveChanges(); + } private void setValue(Record record, int number, float? value) { diff --git a/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs b/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs index b969a89..b36faed 100644 --- a/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs @@ -7,29 +7,32 @@ namespace WebAPI.dataProcessors { public class T3SourceYearSummaryProcessor { - private AppDbContext db; - private SpreadsheetsResource.ValuesResource googleSheetValues; - private LayersController controller; + private readonly AppDbContext db; + private readonly SpreadsheetsResource.ValuesResource googleSheetValues; + private readonly LayersController controller; + private readonly LogsController logsController; public T3SourceYearSummaryProcessor( AppDbContext _db, SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) + LayersController _controller, + LogsController _logsController + ) { db = _db; googleSheetValues = _googleSheetValues; controller = _controller; + logsController = _logsController; } public void process(Layer processWorker) { - string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1; - int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0"); - if (year == 0) + string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; + string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; + if (source == null) { - throw new Exception($"Year is empty, {processWorker!.Name}"); + throw new Exception("Source record not found"); } - Layer? processedLayer = db.Layers .Where(x => x.ParentId == processWorker!.Id && !x.IsDeleted) @@ -48,7 +51,7 @@ namespace WebAPI.dataProcessors ParentId = processWorker!.Id, Number = db.Layers.Count() + 1, }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3"; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedAt = DateTime.UtcNow; @@ -60,29 +63,29 @@ namespace WebAPI.dataProcessors List newRecords = new List(); - List sources = new List(); + List dataSources = new List(); for (int i=1; i<13; i++) { - Layer? source = db.Layers.Where(x => + Layer? dataSource = db.Layers.Where(x => x.Type == LayerType.processed && !x.IsDeleted - && x.Name.Contains($"{year}/{i}-{sourceName}-T3")) + && x.Name.Contains($"{year}/{i}-{source}-T3")) .Include(x => x.Records) .FirstOrDefault(); - if (source != null) + if (dataSource != null) { - sources.Add(source); + dataSources.Add(dataSource!); } } - if (sources.Count == 0) + if (dataSources.Count == 0) { - throw new Exception($"DataSources are empty, {processWorker!.Name}"); + throw new Exception("DataSources are empty"); } - List allRecords = sources.SelectMany(x => x.Records!).ToList(); + List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - foreach (Record baseRecord in sources.Last()?.Records!) + foreach (Record baseRecord in dataSources.Last()?.Records!) { List codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); Record processedRecord = new Record