T3SourceSummary processor

WebAPI/Controllers/LayersController.cs

@@ -42,19 +42,19 @@ namespace WebAPI.Controllers
             try
             {
                 IQueryable<Layer> response = db.Layers.Where(x => !x.IsDeleted);
-                if (name != null)
+                if (name != null)
                 {
-                    response = response.Where(x => x.Name.Contains(name));
+                    response = response.Where(x => x.Name.Contains(name));
                 }
-                if (type != null)
+                if (type != null)
                 {
-                    response = response.Where(x => x.Type == type);
+                    response = response.Where(x => x.Type == type);
                 }
 
                 return Ok(response
                     .OrderByDescending(x => x.Number)
                     .Skip(start).Take(limit).ToList());
-                
+
             }
             catch (Exception e)
             {
@@ -194,14 +194,28 @@ namespace WebAPI.Controllers
                     .Include(x => x.Records)
                     .Where(x =>
                         x.Records!.Any(x => x.Code == "Type" && x.Desc1 == "ProcessWorker") &&
-                        x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True") 
+                        x.Records!.Any(x => x.Code == "IsEnabled" && x.Desc1 == "True")
+                        //&& x.Number == 215
                         )
                     .ToList();
 
                 layersToProcess = new List<Layer>();
-                
+
                 foreach (Layer processWorker in processWorkerLayers)
                 {
+                    string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
+                    if (processType == null)
+                    {
+                        logsController.AddEntry(new LogEntry
+                        {
+                            Title = "Process error",
+                            Type = LogEntryType.error,
+                            LogType = LogType.import,
+                            Message = processWorker?.Name + " ProcessType record not found",
+                            CreatedAt = DateTime.UtcNow
+                        });
+                        return BadRequest();
+                    }
                     string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
                     if (sourceName == null)
                     {
@@ -214,10 +228,21 @@ namespace WebAPI.Controllers
                             CreatedAt = DateTime.UtcNow
                         });
                         return BadRequest();
-                    } else
+                    }
+
+                    if (processType == "T3-SourceSummary")
                     {
+                        if (processWorker != null)
+                        {
+                            T3SourceSummaryProcessor processor = new T3SourceSummaryProcessor(db, googleSheetValues, this);
+                            processor.process(processWorker);
+                        }
+                    }
+                    else
+                    {
+
                         Layer sourceLayer = db.Layers.Include(x => x.Records)
-                            .Single(x => x.Name == sourceName);
+                                .Single(x => x.Name == sourceName);
                         if (sourceLayer == null)
                         {
                             logsController.AddEntry(new LogEntry
@@ -229,55 +254,44 @@ namespace WebAPI.Controllers
                                 CreatedAt = DateTime.UtcNow
                             });
                             return BadRequest();
-                        } else
+                        }
+
+                        string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
+                        string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
+                        var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
+                        var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
+                        if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
                         {
-                            string startDate = sourceLayer.Records!.Where(x => x.Code == "StartDate").First().Desc1!;
+                            switch (processType)
-                            string endDate = sourceLayer.Records!.Where(x => x.Code == "EndDate").First().Desc1!;
-                            var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
-                            var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
-                            if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
                             {
-                                string? processType = processWorker?.Records?.Single(x => x.Code == "ProcessType")?.Desc1;
+                                case "Copy":
-                                if (processType == null)
+                                    CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
-                                {
+                                    cp.process(sourceLayer, processWorker?.Id);
-                                    logsController.AddEntry(new LogEntry
+                                    break;
-                                    {
+                                case "Deaggregation":
-                                        Title = "Process error",
+                                    DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
-                                        Type = LogEntryType.error,
+                                    dp.process(sourceLayer, processWorker?.Id);
-                                        LogType = LogType.import,
+                                    break;
-                                        Message = processWorker?.Name + " ProcessType record not found",
+                                case "T3-SingleSource":
-                                        CreatedAt = DateTime.UtcNow
+                                    T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
-                                    });
+                                    processor.process(sourceLayer, processWorker?.Id);
-                                    return BadRequest();
+                                    break;
-                                } else
-                                {
-                                    switch (processType)
-                                    {
-                                        case "Copy":
-                                            CopyProcessor cp = new CopyProcessor(db, googleSheetValues, this);
-                                            cp.process(sourceLayer, processWorker?.Id);
-                                            break;
-                                        case "Deaggregation":
-                                            DeaggregationProcessor dp = new DeaggregationProcessor(db, googleSheetValues, this);
-                                            dp.process(sourceLayer, processWorker?.Id);
-                                            break;
-                                        case "T3-SingleSource":
-                                            T3SingleSourceProcessor processor = new T3SingleSourceProcessor(db, googleSheetValues, this);
-                                            processor.process(sourceLayer, processWorker?.Id);
-                                            break;
-                                    }
-                                    logsController.AddEntry(new LogEntry
-                                    {
-                                        Title = $"Process Success, {sourceLayer.Name}",
-                                        Type = LogEntryType.info,
-                                        LogType = LogType.process,
-                                        CreatedAt = DateTime.UtcNow
-                                    });
-                                }
                             }
+                            logsController.AddEntry(new LogEntry
+                            {
+                                Title = $"Process Success, {sourceLayer.Name}",
+                                Type = LogEntryType.info,
+                                LogType = LogType.process,
+                                CreatedAt = DateTime.UtcNow
+                            });
+                        } else if (endDateParsed.Date >= DateTime.UtcNow.Date)
+                        {
+                            // TODO: make isEnabled = false - layer is out of date
+                            var name = processWorker.Name;
                         }
                     }
                 }
+
                 return Ok();
             }
             catch (Exception e)
@@ -391,9 +405,9 @@ namespace WebAPI.Controllers
                     layer.Records.Add(record);
                 };
             }
-            
+
             AddLayer(layer, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
-            
+
             logsController.AddEntry(new LogEntry
             {
                 Title = $"Import Success, {importWorker.Name}",
@@ -401,7 +415,7 @@ namespace WebAPI.Controllers
                 LogType = LogType.import,
                 CreatedAt = DateTime.UtcNow
             });
-            
+
         }
         internal Layer AddLayer(Layer input, Guid currentUserId)
         {
@@ -418,11 +432,11 @@ namespace WebAPI.Controllers
         {
             try
             {
-                List<Record> toDelete = db.Records.Where(x => x.LayerId == id).ToList();
+                List<Record> toDelete = db.Records.Where(x => x.LayerId == id).ToList();
-                if (toDelete.Count > 0)
+                if (toDelete.Count > 0)
-                {
+                {
-                    db.Records.RemoveRange(toDelete);
+                    db.Records.RemoveRange(toDelete);
-                }               
+                }
                 foreach (Record record in records)
                 {
                     record.CreatedById = currentUserId;

WebAPI/dataProcessors/t3.SingleSource.processor.cs

@@ -123,7 +123,7 @@ namespace WebAPI.dataProcessors
 
             List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
 
-            foreach (Record baseRecord in dataSources.First()?.Records!)
+            foreach (Record baseRecord in dataSources.Last()?.Records!)
             {
                 List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
 

WebAPI/dataProcessors/t3.SourceSummary.processor.cs

@@ -0,0 +1,110 @@
+using Google.Apis.Sheets.v4;
+using Microsoft.EntityFrameworkCore;
+using WebAPI.Controllers;
+using WebAPI.Models;
+
+namespace WebAPI.dataProcessors
+{
+    public class T3SourceSummaryProcessor
+    {
+        private AppDbContext db;
+        private SpreadsheetsResource.ValuesResource googleSheetValues;
+        private LayersController controller;
+
+        public T3SourceSummaryProcessor(
+            AppDbContext _db,
+            SpreadsheetsResource.ValuesResource _googleSheetValues,
+            LayersController _controller)
+        {
+            db = _db;
+            googleSheetValues = _googleSheetValues;
+            controller = _controller;
+        }
+
+        public void process(Layer processWorker)
+        {
+            string? sourceName = processWorker?.Records?.Single(x => x.Code == "Source")?.Desc1;
+            int year = int.Parse(processWorker?.Records?.Single(x => x.Code == "Year")?.Desc1 ?? "0");
+            if (year == 0)
+            {
+                throw new Exception($"Year is empty, {processWorker!.Name}");
+            }
+
+            Layer? processedLayer = db.Layers
+                                    .Where(x => x.ParentId == processWorker!.Id
+                                                && !x.IsDeleted)
+                                    .OrderByDescending(x => x.CreatedAt)
+                                    .FirstOrDefault();
+
+            bool isNew = false;
+            if (processedLayer == null)
+            {
+                isNew = true;
+                processedLayer = new Layer
+                {
+                    Id = Guid.NewGuid(),
+                    Source = "",
+                    Type = LayerType.processed,
+                    ParentId = processWorker!.Id,
+                    Number = db.Layers.Count() + 1,
+                };
+                processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{sourceName}-T3";
+                processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+                processedLayer.CreatedAt = DateTime.UtcNow;
+                processedLayer.ModifiedAt = DateTime.UtcNow;
+            }
+            processedLayer.Sources = new List<ProcessSource>();
+            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
+            processedLayer.ModifiedAt = DateTime.UtcNow;
+
+            List<Record> newRecords = new List<Record>();
+
+            List<Layer> sources = new List<Layer>();
+            for (int i=1; i<13; i++)
+            {
+                Layer? source = db.Layers.Where(x =>
+                    x.Type == LayerType.processed
+                    && !x.IsDeleted
+                    && x.Name.Contains($"{year}/{i}-{sourceName}-T3"))
+                    .Include(x => x.Records)
+                    .FirstOrDefault();
+                if (source != null)
+                {
+                    sources.Add(source);
+                }
+            }
+
+            if (sources.Count == 0)
+            {
+                throw new Exception($"DataSources are empty, {processWorker!.Name}");
+            }
+
+            List<Record> allRecords = sources.SelectMany(x => x.Records!).ToList();
+
+            foreach (Record baseRecord in sources.Last()?.Records!)
+            {
+                List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
+                Record processedRecord = new Record
+                {
+                    Id = Guid.NewGuid(),
+                    Code = baseRecord.Code,
+                    CreatedAt = DateTime.UtcNow,
+                    ModifiedAt = DateTime.UtcNow,
+                    Value1 = codeRecords.Sum(x => x.Value32)
+                };
+                newRecords.Add(processedRecord);
+            }
+
+            if (isNew)
+            {
+                db.Layers.Add(processedLayer);
+            } else
+            {
+                db.Layers.Update(processedLayer);
+            }
+            controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
+            db.SaveChanges();
+        }
+    }
+}