AutoProcess Refactor. Logging improvement. MultiSourceSummaryProcessor

WebAPI/Controllers/LayersController.cs

@@ -186,20 +186,143 @@ namespace WebAPI.Controllers
                 return Unauthorized();
             }
 
-            List<Layer> processWorkerLayers;
-            List<Layer> layersToProcess;
             try
             {
-                processWorkerLayers = db.Layers
+                List<Layer> processWorkerLayers = db.Layers
                     .Include(x => x.Records)
                     .Where(x =>
                         x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
                         x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
-                        //&& x.Number == 215
+                        //&& x.Number == 262
                         )
+                    .OrderBy(x => x.CreatedAt)
                     .ToList();
 
-                layersToProcess = new List<Layer>();
+                if (processWorkerLayers.Count() == 0)
+                {
+                    logsController.AddEntry(new LogEntry
+                    {
+                        Title = "No Layers to process.",
+                        Type = LogEntryType.info,
+                        LogType = LogType.process,
+                        CreatedAt = DateTime.UtcNow,
+                    });
+                    return Ok();
+                }
+
+                foreach (Layer processWorker in processWorkerLayers)
+                {
+                    try
+                    {
+                        string name = processWorker.Name;
+                        string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
+                        if (year == null)
+                        {
+                            throw new Exception("Year record nod found");
+                        }
+
+                        if (int.Parse(year!) < DateTime.UtcNow.Year)
+                        {
+                            logsController.AddEntry(new LogEntry
+                            {
+                                Title = $"{processWorker!.Name}, {processWorker.Id}",
+                                Type = LogEntryType.warning,
+                                LogType = LogType.process,
+                                Message = "processLayer is out of date. Should be disabled. Not processed.",
+                                CreatedAt = DateTime.UtcNow
+                            });
+                            continue;
+                        }
+
+                        string? processType = processWorker?.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1;
+                        if (processType == null)
+                        {
+                            throw new Exception("ProcessType record not found");
+                        }
+                        if (processType == "T3-SourceYearSummary")
+                        {
+                            if (int.Parse(year!) < DateTime.UtcNow.Year)
+                            {
+                                logsController.AddEntry(new LogEntry
+                                {
+                                    Title = $"{processWorker!.Name}, {processWorker.Id}",
+                                    Type = LogEntryType.warning,
+                                    LogType = LogType.process,
+                                    Message = "processLayer is out of date. Should be disabled. Not processed",
+                                    CreatedAt = DateTime.UtcNow
+                                });
+                                continue;
+                            }
+                            T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this, logsController);
+                            processor.process(processWorker!);
+
+                            logsController.AddEntry(new LogEntry
+                            {
+                                Title = $"{processWorker!.Name}, {processWorker.Id}",
+                                Type = LogEntryType.info,
+                                LogType = LogType.process,
+                                Message = "Success",
+                                CreatedAt = DateTime.UtcNow
+                            });
+                            continue;
+                        }
+                        string? month = processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1;
+                        if (month == null)
+                        {
+                            throw new Exception("Month record not found");
+                        }
+                        if (int.Parse(month!) < DateTime.UtcNow.Month)
+                        {
+                            logsController.AddEntry(new LogEntry
+                            {
+                                Title = $"{processWorker!.Name}, {processWorker.Id}",
+                                Type = LogEntryType.warning,
+                                LogType = LogType.process,
+                                Message = "processLayer is out of date. Should be disabled.",
+                                CreatedAt = DateTime.UtcNow
+                            });
+                        }
+                        switch (processType!)
+                        {
+                            case "T3-SingleSource":
+                                {
+                                    T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
+                                    processor.process(processWorker!);
+                                    break;
+                                }
+                            case "T3-MultiSourceSummary":
+                                {
+                                    T3MultiSourceSummaryProcessor processor = new T3MultiSourceSummaryProcessor(db, googleSheetValues, this);
+                                    processor.process(processWorker!);
+                                    break;
+                                }
+                        }
+                        logsController.AddEntry(new LogEntry
+                        {
+                            Title = $"{processWorker!.Name}, {processWorker.Id}",
+                            Type = LogEntryType.info,
+                            LogType = LogType.process,
+                            Message = "Success",
+                            CreatedAt = DateTime.UtcNow
+                        });
+                    } catch (Exception e)
+                    {
+                        logsController.AddEntry(new LogEntry
+                        {
+                            Title = $"{processWorker!.Name}, {processWorker.Id}",
+                            Type = LogEntryType.error,
+                            LogType = LogType.process,
+                            Message = e.ToString(),
+                            CreatedAt = DateTime.UtcNow
+                        });
+                    }              
+                }
+
+                return Ok();
+
+
+
+
 
                 foreach (Layer processWorker in processWorkerLayers)
                 {
@@ -216,8 +339,8 @@ namespace WebAPI.Controllers
                         });
                         return BadRequest();
                     }
-                    string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
+                    List<Record>? sources = processWorker?.Records?.Where(x => x.Code == "Source").ToList();
-                    if (sourceName == null)
+                    if (sources != null && sources.Count() == 0)
                     {
                         logsController.AddEntry(new LogEntry
                         {
@@ -234,15 +357,17 @@ namespace WebAPI.Controllers
                     {
                         if (processWorker != null)
                         {
-                            T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
+                            //T3SourceYearSummaryProcessor processor = new T3SourceYearSummaryProcessor(db, googleSheetValues, this);
-                            processor.process(processWorker);
+                           // processor.process(processWorker);
                         }
-                    }
+                    } else if (processType == "T3-MultiSourceSummary")
-                    else
+                    {
+                        var tst = 5;
+                    } else
                     {
 
                         Layer sourceLayer = db.Layers.Include(x => x.Records)
-                                .Single(x => x.Name == sourceName);
+                                .Single(x => x.Name == sources!.First().Desc1);
                         if (sourceLayer == null)
                         {
                             logsController.AddEntry(new LogEntry
@@ -262,21 +387,7 @@ namespace WebAPI.Controllers
                         var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
                         if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
                         {
-                            switch (processType)
+
-                            {
-                                case "Copy":
-                                    CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
-                                    cp.process(sourceLayer, processWorker?.Id);
-                                    break;
-                                case "Deaggregation":
-                                    DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
-                                    dp.process(sourceLayer, processWorker?.Id);
-                                    break;
-                                case "T3-SingleSource":
-                                    T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
-                                    processor.process(sourceLayer, processWorker?.Id);
-                                    break;
-                            }
                             logsController.AddEntry(new LogEntry
                             {
                                 Title = $"Process Success, {sourceLayer.Name}",

WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs

@@ -0,0 +1,116 @@
+using Google.Apis.Sheets.v4;
+using Microsoft.EntityFrameworkCore;
+using WebAPI.Controllers;
+using WebAPI.Models;
+
+namespace WebAPI.dataProcessors
+{
+    public class T3MultiSourceSummaryProcessor
+    {
+        private AppDbContext db;
+        private SpreadsheetsResource.ValuesResource googleSheetValues;
+        private LayersController controller;
+
+        public T3MultiSourceSummaryProcessor(
+            AppDbContext _db,
+            SpreadsheetsResource.ValuesResource _googleSheetValues,
+            LayersController _controller)
+        {
+            db = _db;
+            googleSheetValues = _googleSheetValues;
+            controller = _controller;
+        }
+
+        public void process(Layer processWorker)
+        {
+            int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
+            int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
+            List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
+            if (sources!.Count() == 0)
+            {
+                throw new Exception("Source record not found");
+            }
+
+            Layer? processedLayer = db.Layers
+                                    .Where(x => x.ParentId == processWorker!.Id
+                                                && !x.IsDeleted)
+                                    .OrderByDescending(x => x.CreatedAt)
+                                    .FirstOrDefault();
+
+            bool isNew = false;
+            if (processedLayer == null)
+            {
+                isNew = true;
+                processedLayer = new Layer
+                {
+                    Id = Guid.NewGuid(),
+                    Source = "",
+                    Type = LayerType.processed,
+                    ParentId = processWorker!.Id,
+                    Number = db.Layers.Count() + 1,
+                };
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3";
+                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.CreatedAt = DateTime.UtcNow;
+                processedLayer.ModifiedAt = DateTime.UtcNow;
+            }
+            processedLayer.Sources = new List<ProcessSource>();
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            List<Record> newRecords = new List<Record>();
+
+            List<Layer> dataSources = new List<Layer>();
+
+            foreach (Record source in sources!)
+            {
+                Layer? dataSource = db.Layers.Where(x =>
+                    x.Type == LayerType.processed &&
+                    !x.IsDeleted &&
+                    x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
+                    )
+                    .Include(x => x.Records)
+                    .FirstOrDefault();
+                if (dataSource != null)
+                {
+                    dataSources.Add(dataSource);
+                }
+            }
+
+            if (dataSources.Count == 0)
+            {
+                throw new Exception($"DataSources are empty");
+            }
+
+            List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
+            foreach (Record baseRecord in dataSources.Last()?.Records!)
+            {
+
+                List<Record> codeRecords = allRecords.Where(x =>
+                    x.Code!.Substring(1) == baseRecord.Code!.Substring(1))
+                    .ToList();
+                Record processedRecord = new Record
+                {
+                    Id = Guid.NewGuid(),
+                    Code = $"9{baseRecord.Code!.Substring(1)}",
+                    CreatedAt = DateTime.UtcNow,
+                    ModifiedAt = DateTime.UtcNow,
+                    Value1 = codeRecords.Sum(x => x.Value32)
+                };
+                newRecords.Add(processedRecord);
+            }
+
+            if (isNew)
+            {
+                db.Layers.Add(processedLayer);
+            }
+            else
+            {
+                db.Layers.Update(processedLayer);
+            }
+            controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
+            db.SaveChanges();
+        }
+    }
+}

WebAPI/dataProcessors/t3.SingleSource.processor.cs

@@ -24,55 +24,23 @@ namespace WebAPI.dataProcessors
             controller = _controller;
         }
 
-        public void process(Layer sourceImportWorker, Guid? parentId)
+        public void process(Layer processWorker)
         {
-            string? sheetId = sourceImportWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
+            int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
-            if (sheetId == null)
+            int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
-            {
+            string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
-                throw new Exception($"SheetId not found, {sourceImportWorker.Name}");
+            if (source == null)
-            }
+            {
-            string? sheetTabName = sourceImportWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
+                throw new Exception("Source record not found");
-            if (sheetId == null)
-            {
-                throw new Exception($"SheetTabName not found, {sourceImportWorker.Name}");
-            }
-            string? importYearCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
-            if (importYearCell == null)
-            {
-                throw new Exception($"ImportYear not found, {sourceImportWorker.Name}");
-            }
-            string? importMonthCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
-            if (importMonthCell == null)
-            {
-                throw new Exception($"ImportMonth not found, {sourceImportWorker.Name}");
-            }
-            string? importNameCell = sourceImportWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
-            if (importNameCell == null)
-            {
-                throw new Exception($"ImportName not found, {sourceImportWorker.Name}");
             }
-            var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
+            Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == source);
-            string? name = nameResponse.Values[0][0].ToString();
+            if (sourceImportWorker == null)
-            if (name == null)
+            {
-            {
+                throw new Exception("SourceImportWorkerL layer not found");
-                throw new Exception($"ImportName cell is empty, {sourceImportWorker.Name}");
-            }
-            var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
-            int? year = int.Parse(yearResponse.Values[0][0].ToString() ?? "");
-            if (year == null)
-            {
-                throw new Exception($"ImportYear cell is empty, {sourceImportWorker.Name}");
-            }
-            var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
-            string? month = monthResponse.Values[0][0].ToString();
-            if (month == null)
-            {
-                throw new Exception($"ImportMonth cell is empty, {sourceImportWorker.Name}");
             }
 
-
             Layer? processedLayer = db.Layers
-                                    .Where(x => x.ParentId == parentId)
+                                    .Where(x => x.ParentId == processWorker!.Id)
                                     .OrderByDescending(x => x.CreatedAt)
                                     .FirstOrDefault();
 
@@ -85,10 +53,10 @@ namespace WebAPI.dataProcessors
                     Id = Guid.NewGuid(),
                     Source = "",
                     Type = LayerType.processed,
-                    ParentId = parentId,
+                    ParentId = processWorker!.Id,
                     Number = db.Layers.Count() + 1,
                 };
-                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{name}-T3";
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
                 processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                 processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                 processedLayer.CreatedAt = DateTime.UtcNow;
@@ -98,6 +66,7 @@ namespace WebAPI.dataProcessors
             processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
             processedLayer.ModifiedAt = DateTime.UtcNow;
 
+            
             List<Record> newRecords = new List<Record>();
 
             List<Layer> dataSources = db.Layers
@@ -110,16 +79,7 @@ namespace WebAPI.dataProcessors
             {
                 throw new Exception($"DataSources are empty, {sourceImportWorker.Name}");
             }
-            /*
+
-            foreach (Layer source in dataSources)
-            {
-                response.Sources.Add(new ProcessSource
-                {
-                    LayerId = response.Id,
-                    SourceId = source.Id
-                });
-            }
-            */
 
             List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
 
@@ -135,11 +95,11 @@ namespace WebAPI.dataProcessors
                     ModifiedAt = DateTime.UtcNow
                 };
 
-                int lastDayInMonth = DateTime.DaysInMonth(year ?? 0, int.Parse(month ?? ""));
+                int lastDayInMonth = DateTime.DaysInMonth(year, month);
                 float previousValue = 0;
                 //day 1
                 float firstVal = codeRecords
-                    .Where(x => x.CreatedAt.Date <= new DateTime(year ?? 0, int.Parse(month ?? ""), 1))
+                    .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
                     .OrderByDescending(x => x.CreatedAt)
                     .FirstOrDefault()?.Value1 ?? 0;
                 setValue(processedRecord, 1, firstVal);
@@ -148,7 +108,7 @@ namespace WebAPI.dataProcessors
                 for (int i=2; i<lastDayInMonth; i++)
                 {
                     float? dayVal = codeRecords
-                        .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == int.Parse(month ?? ""))
+                        .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month)
                         .OrderByDescending(x => x.CreatedAt)
                         .FirstOrDefault()?.Value1;
                     if (dayVal == null)
@@ -164,7 +124,7 @@ namespace WebAPI.dataProcessors
                 }
                 //last day
                 float? lastVal = codeRecords
-                    .Where(x => x.CreatedAt.Date >= new DateTime(year ?? 0, int.Parse(month ?? ""), lastDayInMonth))
+                    .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth))
                     .OrderByDescending(x => x.CreatedAt)
                     .FirstOrDefault()?.Value1;
 
@@ -194,6 +154,7 @@ namespace WebAPI.dataProcessors
             }
             controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
             db.SaveChanges();
+            
         }
         private void setValue(Record record, int number, float? value)
         {

WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs

@@ -7,29 +7,32 @@ namespace WebAPI.dataProcessors
 {
     public class T3SourceYearSummaryProcessor
     {
-        private AppDbContext db;
+        private readonly AppDbContext db;
-        private SpreadsheetsResource.ValuesResource googleSheetValues;
+        private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
-        private LayersController controller;
+        private readonly LayersController controller;
+        private readonly LogsController logsController;
 
         public T3SourceYearSummaryProcessor(
             AppDbContext _db,
             SpreadsheetsResource.ValuesResource _googleSheetValues,
-            LayersController _controller)
+            LayersController _controller,
+            LogsController _logsController
+            )
         {
             db = _db;
             googleSheetValues = _googleSheetValues;
             controller = _controller;
+            logsController = _logsController;
         }
 
         public void process(Layer processWorker)
         {
-            string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
+            string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
-            int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0");
+            string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
-            if (year == 0)
+            if (source == null)
             {
-                throw new Exception($"Year is empty, {processWorker!.Name}");
+                throw new Exception("Source record not found");
             }
-
             Layer? processedLayer = db.Layers
                                     .Where(x => x.ParentId == processWorker!.Id
                                                 && !x.IsDeleted)
@@ -48,7 +51,7 @@ namespace WebAPI.dataProcessors
                     ParentId = processWorker!.Id,
                     Number = db.Layers.Count() + 1,
                 };
-                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3";
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
                 processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                 processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
                 processedLayer.CreatedAt = DateTime.UtcNow;
@@ -60,29 +63,29 @@ namespace WebAPI.dataProcessors
 
             List<Record> newRecords = new List<Record>();
 
-            List<Layer> sources = new List<Layer>();
+            List<Layer> dataSources = new List<Layer>();
             for (int i=1; i<13; i++)
             {
-                Layer? source = db.Layers.Where(x =>
+                Layer? dataSource = db.Layers.Where(x =>
                     x.Type == LayerType.processed
                     && !x.IsDeleted
-                    && x.Name.Contains($"{year}/{i}-{sourceName}-T3"))
+                    && x.Name.Contains($"{year}/{i}-{source}-T3"))
                     .Include(x => x.Records)
                     .FirstOrDefault();
-                if (source != null)
+                if (dataSource != null)
                 {
-                    sources.Add(source);
+                    dataSources.Add(dataSource!);
                 }
             }
 
-            if (sources.Count == 0)
+            if (dataSources.Count == 0)
             {
-                throw new Exception($"DataSources are empty, {processWorker!.Name}");
+                throw new Exception("DataSources are empty");
             }
 
-            List<Record> allRecords = sources.SelectMany(x => x.Records!).ToList();
+            List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
 
-            foreach (Record baseRecord in sources.Last()?.Records!)
+            foreach (Record baseRecord in dataSources.Last()?.Records!)
             {
                 List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
                 Record processedRecord = new Record