From 763b4a8c4d47486ef4a9f7ca7030df6f4ba40320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieliski?= Date: Tue, 18 Jun 2024 22:02:19 +0200 Subject: [PATCH] Code refactor finished --- WebAPI/Controllers/AdminController.cs | 8 +- WebAPI/Controllers/LayersController.cs | 120 +++--- WebAPI/Controllers/LogsController.cs | 8 +- WebAPI/Exports/googleSheet.export.cs | 6 +- WebAPI/Models/Layer.cs | 78 ++-- WebAPI/Models/LogEntry.cs | 41 +-- WebAPI/Models/ProcessSource.cs | 23 +- WebAPI/Models/Record.cs | 14 +- WebAPI/dataImporters/morska.fk2.importer.cs | 3 +- WebAPI/dataImporters/morska.importer.cs | 3 +- WebAPI/dataProcessors/t1.r1.processor.cs | 17 +- ....MultiSourceCopySelectedCodes.processor.cs | 183 +++++---- ...eCopySelectedCodesYearSummary.processor.cs | 209 +++++------ .../t3.MultiSourceSummary.processor.cs | 295 +++++++-------- .../t3.MultiSourceYearSummary.processor.cs | 347 ++++++++---------- .../t3.SingleSource.processor.cs | 253 ++++++------- .../t3.SourceYearSummary.processor.cs | 209 +++++------ .../t4.SingleSource.processor.cs | 205 +++++------ WebAPI/dataProcessors/t4.r2.processor.cs | 311 ++++++++-------- 19 files changed, 1105 insertions(+), 1228 deletions(-) diff --git a/WebAPI/Controllers/AdminController.cs b/WebAPI/Controllers/AdminController.cs index 90fb240..3344a92 100644 --- a/WebAPI/Controllers/AdminController.cs +++ b/WebAPI/Controllers/AdminController.cs @@ -85,8 +85,8 @@ public class AdminController : Controller _logsController.AddEntry(new LogEntry { Title = "Backup success", - Type = LogEntryType.info, - LogType = LogType.backup, + Type = LogEntryType.Info, + LogType = LogType.Backup, CreatedAt = DateTime.UtcNow }); return Ok(); @@ -96,8 +96,8 @@ public class AdminController : Controller _logsController.AddEntry(new LogEntry { Title = "Backup error", - Type = LogEntryType.error, - LogType = LogType.backup, + Type = LogEntryType.Error, + LogType = LogType.Backup, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index 7719d78..83da2fc 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -90,8 +90,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - wrong apiKey ({number})", - Type = LogEntryType.warning, - LogType = LogType.powerBI, + Type = LogEntryType.Warning, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); @@ -105,8 +105,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - no authorization header ({number})", - Type = LogEntryType.warning, - LogType = LogType.powerBI, + Type = LogEntryType.Warning, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); @@ -118,8 +118,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - wrong auth header format ({number})", - Type = LogEntryType.warning, - LogType = LogType.powerBI, + Type = LogEntryType.Warning, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); @@ -133,8 +133,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Unauthorized request - bad credentials ({number})", - Type = LogEntryType.warning, - LogType = LogType.powerBI, + Type = LogEntryType.Warning, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return Unauthorized(); @@ -143,8 +143,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Sending data for layer {number}", - Type = LogEntryType.info, - LogType = LogType.powerBI, + Type = LogEntryType.Info, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); @@ -157,8 +157,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = e.ToString(), - Type = LogEntryType.error, - LogType = LogType.powerBI, + Type = LogEntryType.Error, + LogType = LogType.PowerBi, CreatedAt = DateTime.UtcNow }); return BadRequest(e.ToString()); @@ -213,8 +213,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = "No Layers to import.", - Type = LogEntryType.info, - LogType = LogType.import, + Type = LogEntryType.Info, + LogType = LogType.Import, CreatedAt = DateTime.UtcNow }); return Ok(); @@ -235,8 +235,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.import, + Type = LogEntryType.Info, + LogType = LogType.Import, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -267,8 +267,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.import, + Type = LogEntryType.Info, + LogType = LogType.Import, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -282,8 +282,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.import, + Type = LogEntryType.Warning, + LogType = LogType.Import, Message = "Success (reimported)", CreatedAt = DateTime.UtcNow }); @@ -293,8 +293,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.import, + Type = LogEntryType.Warning, + LogType = LogType.Import, Message = "importLayer records are up of date. Not processed.", CreatedAt = DateTime.UtcNow }); @@ -306,8 +306,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.error, - LogType = LogType.import, + Type = LogEntryType.Error, + LogType = LogType.Import, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); @@ -321,8 +321,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = "Process error", - Type = LogEntryType.error, - LogType = LogType.import, + Type = LogEntryType.Error, + LogType = LogType.Import, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); @@ -363,8 +363,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"Processing: {type}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, CreatedAt = DateTime.UtcNow }); @@ -383,8 +383,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = "No Layers to process.", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, CreatedAt = DateTime.UtcNow }); } @@ -400,8 +400,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.error, - LogType = LogType.process, + Type = LogEntryType.Error, + LogType = LogType.Process, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); @@ -413,8 +413,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = "Process error", - Type = LogEntryType.error, - LogType = LogType.process, + Type = LogEntryType.Error, + LogType = LogType.Process, Message = e.ToString(), CreatedAt = DateTime.UtcNow }); @@ -445,14 +445,14 @@ public class LayersController : Controller case "T3-SourceYearSummary": { var processor = - new T3SourceYearSummaryProcessor(_db, _googleSheetValues, this); - processor.process(processWorker); + new T3SourceYearSummaryProcessor(_db, this); + processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -461,14 +461,14 @@ public class LayersController : Controller case "T3-MultiSourceYearSummary": { var processor = - new T3MultiSourceYearSummaryProcessor(_db, _googleSheetValues, this, _logsController); - processor.process(processWorker); + new T3MultiSourceYearSummaryProcessor(_db, this, _logsController); + processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -477,14 +477,14 @@ public class LayersController : Controller case "T3-MultiSourceCopySelectedCodesYearSummary": { var processor = - new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, _googleSheetValues, this); - processor.process(processWorker); + new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this); + processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -498,8 +498,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -507,14 +507,14 @@ public class LayersController : Controller } case "T4-R2": { - var processor = new T4R2Processor(_db, _googleSheetValues, this, _logsController); + var processor = new T4R2Processor(_db, this, _logsController); processor.Process(processWorker); _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); @@ -531,28 +531,28 @@ public class LayersController : Controller switch (processType!) { case "T3-SingleSource": - var t3SingleSource = new T3SingleSourceProcessor(_db, _googleSheetValues, this); + var t3SingleSource = new T3SingleSourceProcessor(_db, this); { - t3SingleSource.process(processWorker); + t3SingleSource.Process(processWorker); break; } case "T4-SingleSource": { - var t4SingleSource = new T4SingleSourceProcessor(_db, _googleSheetValues, this); - t4SingleSource.process(processWorker); + var t4SingleSource = new T4SingleSourceProcessor(_db, this); + t4SingleSource.Process(processWorker); break; } case "T3-MultiSourceSummary": { var t3MultiSourceSummary = - new T3MultiSourceSummaryProcessor(_db, _googleSheetValues, this, _logsController); - t3MultiSourceSummary.process(processWorker); + new T3MultiSourceSummaryProcessor(_db, this, _logsController); + t3MultiSourceSummary.Process(processWorker); break; } case "T3-MultiSourceCopySelectedCodes": { var t3MultiSourceCopySelectedCode = - new T3MultiSourceCopySelectedCodesProcessor(_db, _googleSheetValues, this); + new T3MultiSourceCopySelectedCodesProcessor(_db, this); t3MultiSourceCopySelectedCode.Process(processWorker); break; } @@ -561,8 +561,8 @@ public class LayersController : Controller _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.info, - LogType = LogType.process, + Type = LogEntryType.Info, + LogType = LogType.Process, Message = "Success", CreatedAt = DateTime.UtcNow }); diff --git a/WebAPI/Controllers/LogsController.cs b/WebAPI/Controllers/LogsController.cs index df0cc4c..3acfea8 100644 --- a/WebAPI/Controllers/LogsController.cs +++ b/WebAPI/Controllers/LogsController.cs @@ -28,10 +28,10 @@ public class LogsController : Controller var type = entry.LogType switch { - LogType.import => "Import", - LogType.backup => "Backup", - LogType.process => "Process", - LogType.powerBI => "PowerBIAccess", + LogType.Import => "Import", + LogType.Backup => "Backup", + LogType.Process => "Process", + LogType.PowerBi => "PowerBIAccess", _ => "Other" }; var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute(); diff --git a/WebAPI/Exports/googleSheet.export.cs b/WebAPI/Exports/googleSheet.export.cs index fa1ef72..03f2cf0 100644 --- a/WebAPI/Exports/googleSheet.export.cs +++ b/WebAPI/Exports/googleSheet.export.cs @@ -32,19 +32,19 @@ public class GoogleSheetExport switch (layer.Type) { - case LayerType.import: + case LayerType.Import: { data.Add(new List { "Code", "Value1" }); data.AddRange(layer.Records!.Select(record => new List { record.Code!, record.Value1! })); break; } - case LayerType.administration: + case LayerType.Administration: { data.Add(new List { "Code", "Desc1"}); data.AddRange(layer.Records!.Select(record => new List { record.Code!, record.Desc1! })); break; } - case LayerType.processed: + case LayerType.Processed: { data.Add(new List { "Code", "Value1", "Value2", "Value3", "Value3", "Value5", "Value6", "Value7", "Value8", "Value9", "Value10", diff --git a/WebAPI/Models/Layer.cs b/WebAPI/Models/Layer.cs index cce55b9..0dedcba 100644 --- a/WebAPI/Models/Layer.cs +++ b/WebAPI/Models/Layer.cs @@ -1,44 +1,42 @@ using System.ComponentModel.DataAnnotations; -namespace WebAPI.Models +namespace WebAPI.Models; + +public enum LayerType { - public enum LayerType - { - import, - processed, - administration, - } - public class Layer - { - #region Properties - [Key] - public Guid Id { get; set; } - [Required] - public int Number { get; set; } - [Required] - public string? Source { get; set; } - [Required] - public string? Name { get; set; } - [Required] - public LayerType Type { get; set; } - [Required] - public DateTime CreatedAt { get; set; } - [Required] - public DateTime ModifiedAt { get; set; } - [Required] - public bool IsDeleted { get; set; } - #endregion - #region Relations - public ICollection? Records { get; set; } - public ICollection? Sources { get; set; } - [Required] - public Guid CreatedById { get; set; } - public User? CreatedBy { get; set; } - [Required] - public Guid ModifiedById { get; set; } - public User? ModifiedBy { get; set; } - public Guid? ParentId { get; set; } - public Layer? Parent { get; set; } - #endregion - } + Import, + Processed, + Administration } +public class Layer +{ + #region Properties + [Key] + public Guid Id { get; init; } + [Required] + public int Number { get; init; } + + [Required] + [MaxLength(50)] + public string? Name { get; set; } + [Required] + public LayerType Type { get; init; } + [Required] + public DateTime CreatedAt { get; set; } + [Required] + public DateTime ModifiedAt { get; set; } + [Required] + public bool IsDeleted { get; init; } + #endregion + #region Relations + public ICollection? Records { get; init; } + [Required] + public Guid CreatedById { get; set; } + public User? CreatedBy { get; init; } + [Required] + public Guid ModifiedById { get; set; } + public User? ModifiedBy { get; init; } + public Guid? ParentId { get; init; } + public Layer? Parent { get; init; } + #endregion +} \ No newline at end of file diff --git a/WebAPI/Models/LogEntry.cs b/WebAPI/Models/LogEntry.cs index 54e227e..f1e2852 100644 --- a/WebAPI/Models/LogEntry.cs +++ b/WebAPI/Models/LogEntry.cs @@ -1,25 +1,22 @@ -using System.ComponentModel.DataAnnotations; +namespace WebAPI.Models; -namespace WebAPI.Models +public enum LogEntryType { - public enum LogEntryType - { - info, - warning, - error, - } - public enum LogType { - import, - backup, - process, - powerBI - } - public class LogEntry - { - public LogType LogType { get; set; } - public LogEntryType Type { get; set; } - public string? Message { get; set; } - public string? Title {get; set;} - public DateTime CreatedAt { get; set; } - } + Info, + Warning, + Error } +public enum LogType { + Import, + Backup, + Process, + PowerBi +} +public class LogEntry +{ + public LogType LogType { get; init; } + public LogEntryType Type { get; init; } + public string? Message { get; init; } + public string? Title {get; init;} + public DateTime CreatedAt { get; init; } +} \ No newline at end of file diff --git a/WebAPI/Models/ProcessSource.cs b/WebAPI/Models/ProcessSource.cs index cd4589c..bbea268 100644 --- a/WebAPI/Models/ProcessSource.cs +++ b/WebAPI/Models/ProcessSource.cs @@ -1,15 +1,14 @@ using System.ComponentModel.DataAnnotations; -namespace WebAPI.Models +namespace WebAPI.Models; + +public class ProcessSource { - public class ProcessSource - { - #region Relations - [Required] - public Guid LayerId { get; set; } - [Required] - public Guid SourceId { get; set; } - public Layer? Source { get; set; } - #endregion - } -} + #region Relations + [Required] + public Guid LayerId { get; init; } + [Required] + public Guid SourceId { get; init; } + public Layer? Source { get; init; } + #endregion +} \ No newline at end of file diff --git a/WebAPI/Models/Record.cs b/WebAPI/Models/Record.cs index dc085c9..4f8bf8f 100644 --- a/WebAPI/Models/Record.cs +++ b/WebAPI/Models/Record.cs @@ -6,10 +6,10 @@ public class Record { #region Properties [Key] - public Guid Id { get; set; } + public Guid Id { get; init; } [Required] [StringLength(50)] - public string? Code { get; set; } + public string? Code { get; init; } public double? Value1 { get; set; } public double? Value2 { get; set; } public double? Value3 { get; set; } @@ -44,15 +44,15 @@ public class Record public double? Value32 { get; set; } //Description fields [StringLength(50)] - public string? Desc1 { get; set; } + public string? Desc1 { get; init; } [StringLength(50)] - public string? Desc2 { get; set; } + public string? Desc2 { get; init; } [StringLength(50)] - public string? Desc3 { get; set; } + public string? Desc3 { get; init; } [StringLength(50)] - public string? Desc4 { get; set; } + public string? Desc4 { get; init; } [StringLength(50)] - public string? Desc5 { get; set; } + public string? Desc5 { get; init; } [StringLength(50)] public DateTime CreatedAt { get; set; } public DateTime ModifiedAt { get; set; } diff --git a/WebAPI/dataImporters/morska.fk2.importer.cs b/WebAPI/dataImporters/morska.fk2.importer.cs index d232c38..7ec704c 100644 --- a/WebAPI/dataImporters/morska.fk2.importer.cs +++ b/WebAPI/dataImporters/morska.fk2.importer.cs @@ -80,10 +80,9 @@ public class MorskaFk2Importer } var layer = new Layer { - Source = "GoogleSheet", Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, - Type = LayerType.import, + Type = LayerType.Import, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, diff --git a/WebAPI/dataImporters/morska.importer.cs b/WebAPI/dataImporters/morska.importer.cs index 34c02d9..70aa747 100644 --- a/WebAPI/dataImporters/morska.importer.cs +++ b/WebAPI/dataImporters/morska.importer.cs @@ -80,10 +80,9 @@ public class MorskaImporter } var layer = new Layer { - Source = "GoogleSheet", Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, - Type = LayerType.import, + Type = LayerType.Import, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, diff --git a/WebAPI/dataProcessors/t1.r1.processor.cs b/WebAPI/dataProcessors/t1.r1.processor.cs index 529b4e3..04cfb3d 100644 --- a/WebAPI/dataProcessors/t1.r1.processor.cs +++ b/WebAPI/dataProcessors/t1.r1.processor.cs @@ -55,8 +55,7 @@ public class T1R1Processor processedLayer = new Layer { Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, + Type = LayerType.Processed, ParentId = processWorker.Id, Number = _db.Layers.Count() + 1 }; @@ -66,7 +65,7 @@ public class T1R1Processor processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; } - processedLayer.Sources = new List(); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; @@ -78,7 +77,7 @@ public class T1R1Processor { var monthCopy = month; var dataSource = _db.Layers.Where(x => - x.Type == LayerType.processed && + x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T3") ) @@ -93,8 +92,8 @@ public class T1R1Processor _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, + Type = LogEntryType.Warning, + LogType = LogType.Process, Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", CreatedAt = DateTime.UtcNow }); @@ -102,7 +101,7 @@ public class T1R1Processor } // year summary var dataSourceSum = _db.Layers.Where(x => - x.Type == LayerType.processed && + x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3") ) @@ -117,8 +116,8 @@ public class T1R1Processor _logsController.AddEntry(new LogEntry { Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, + Type = LogEntryType.Warning, + LogType = LogType.Process, Message = $"Data source {year}/13-{source.Desc1}-T3 not found", CreatedAt = DateTime.UtcNow }); diff --git a/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs index 74b2cfc..c172d47 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs @@ -1,113 +1,106 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors -{ - public class T3MultiSourceCopySelectedCodesProcessor - { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; +namespace WebAPI.dataProcessors; - public T3MultiSourceCopySelectedCodesProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) +public class T3MultiSourceCopySelectedCodesProcessor +{ + private readonly AppDbContext _db; + private readonly LayersController _controller; + + public T3MultiSourceCopySelectedCodesProcessor( + AppDbContext db, + LayersController controller) + { + _db = db; + _controller = controller; + } + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (!sources!.Any()) { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; + throw new Exception("Source record not found"); + } + var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; + if (codes == null) + { + throw new Exception("Codes record not found"); } - public void Process(Layer processWorker) + var codesList = ProcessHelper.ParseCodes(codes); + + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) { - var year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (!sources!.Any()) + isNew = true; + processedLayer = new Layer { - throw new Exception("Source record not found"); - } - var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; - if (codes == null) + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")) + .Include(x => x.Records) + .FirstOrDefault()) + .OfType() + .ToList(); + if (dataSources.Count == 0) + { + throw new Exception("DataSources are empty"); + } + + + var newRecords = dataSources + .SelectMany(x => x.Records!) + .Where(x => codesList.Contains(int.Parse(x.Code!))) + .Select(x => { - throw new Exception("Codes record not found"); - } - - var codesList = ProcessHelper.ParseCodes(codes); - - var processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer + var newRecord = new Record { Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1 + Code = x.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dataSources = sources!.Select(source => db.Layers.Where(x => x.Type == LayerType.processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")) - .Include(x => x.Records) - .FirstOrDefault()) - .OfType() - .ToList(); - if (dataSources.Count == 0) - { - throw new Exception($"DataSources are empty"); - } - - - var newRecords = dataSources - .SelectMany(x => x.Records!) - .Where(x => codesList.Contains(int.Parse(x.Code!))) - .Select(x => + for (var i = 1; i < 33; i++) { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); - } - return newRecord; - }) - .ToList(); - if (isNew) - { - db.Layers.Add(processedLayer); - } - else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); + ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); + } + return newRecord; + }) + .ToList(); + if (isNew) + { + _db.Layers.Add(processedLayer); } + else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); } -} +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs index 8336615..906d92e 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs @@ -1,116 +1,109 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors +namespace WebAPI.dataProcessors; + +public class T3MultiSourceCopySelectedCodesYearSummaryProcessor { - public class T3MultiSourceCopySelectedCodesYearSummaryProcessor + private readonly AppDbContext _db; + private readonly LayersController _controller; + + public T3MultiSourceCopySelectedCodesYearSummaryProcessor( + AppDbContext db, + LayersController controller) { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; - - public T3MultiSourceCopySelectedCodesYearSummaryProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) - { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - } - - public void process(Layer processWorker) - { - int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - List newRecords = new List(); - - List dataSources = new List(); - - for (int i = 1; i < 13; i++) - { - Layer? dataSource = db.Layers.Where(x => - x.Type == LayerType.processed - && !x.IsDeleted - && x.Name != null && x.Name.Contains($"{year}/{i}-AB-T3")) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource!); - } - } - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (Record baseRecord in dataSources.Last()?.Records!) - { - List codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - Record processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - - newRecords.Add(processedRecord); - } - - if (isNew) - { - db.Layers.Add(processedLayer); - } - else - { - db.Layers.Update(processedLayer); - } - - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); - } + _db = db; + _controller = controller; } -} + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + var newRecords = new List(); + + var dataSources = new List(); + + for (var i = 1; i < 13; i++) + { + var i1 = i; + var dataSource = _db.Layers.Where(x => + x.Type == LayerType.Processed + && !x.IsDeleted + && x.Name != null && x.Name.Contains($"{year}/{i1}-AB-T3")) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSource != null) + { + dataSources.Add(dataSource); + } + } + + if (dataSources.Count == 0) + { + throw new Exception("DataSources are empty"); + } + + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + foreach (var baseRecord in dataSources.Last().Records!) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + for (var i = 1; i < 33; i++) + { + ProcessHelper.SetValue(processedRecord, i, + codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + } + + newRecords.Add(processedRecord); + } + + if (isNew) + { + _db.Layers.Add(processedLayer); + } + else + { + _db.Layers.Update(processedLayer); + } + + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); + } +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs index f524f3a..3dae937 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs @@ -1,192 +1,175 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; using Microsoft.EntityFrameworkCore; using WebAPI.Calculator; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors -{ - public class T3MultiSourceSummaryProcessor - { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; - private readonly LogsController logsController; +namespace WebAPI.dataProcessors; - public T3MultiSourceSummaryProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller, - LogsController _logsController) +public class T3MultiSourceSummaryProcessor +{ + private readonly AppDbContext _db; + private readonly LayersController _controller; + private readonly LogsController _logsController; + + public T3MultiSourceSummaryProcessor( + AppDbContext db, + LayersController controller, + LogsController logsController) + { + _db = db; + _controller = controller; + _logsController = logsController; + } + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (!sources!.Any()) { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - logsController = _logsController; + throw new Exception("Source record not found"); } - public void process(Layer processWorker) + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) { - int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - List? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count() == 0) + isNew = true; + processedLayer = new Layer { - throw new Exception("Source record not found"); - } - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + } - List newRecords = new List(); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; - List dataSources = new List(); + var newRecords = new List(); - foreach (Record source in sources!) - { - Layer? dataSource = db.Layers.Where(x => - x.Type == LayerType.processed && - !x.IsDeleted && - x.Name!=null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3") - ) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource); - } - } + var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")) + .Include(x => x.Records) + .FirstOrDefault()) + .OfType() + .ToList(); - if (dataSources.Count == 0) - { - throw new Exception($"DataSources are empty"); - } + if (dataSources.Count == 0) + { + throw new Exception("DataSources are empty"); + } - List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - List baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); - foreach (string baseCode in baseCodess) - { + foreach (var baseCode in baseCodes) + { - List codeRecords = allRecords.Where(x => - x.Code!.Substring(1) == baseCode) - .ToList(); - Record processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"9{baseCode}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i<33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - newRecords.Add(processedRecord); - } - - // Dynamic Codes - List? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); - if (dynamicCodes != null && dynamicCodes.Any()) + var codeRecords = allRecords.Where(x => + x.Code![1..] == baseCode) + .ToList(); + var processedRecord = new Record { - foreach (Record dynamicCode in dynamicCodes) + Id = Guid.NewGuid(), + Code = $"9{baseCode}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + for (var i = 1; i<33; i++) + { + ProcessHelper.SetValue(processedRecord, i, + codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + } + newRecords.Add(processedRecord); + } + + // Dynamic Codes + var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); + if (dynamicCodes != null && dynamicCodes.Any()) + { + foreach (var dynamicCode in dynamicCodes) + { + try { + if (dynamicCode.Desc1 == null) + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula in Record {dynamicCode.Id} is missing.", + CreatedAt = DateTime.UtcNow + }); + continue; + } + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", + CreatedAt = DateTime.UtcNow + }); + continue; + } + try { - if (dynamicCode.Desc1 == null) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula in Record {dynamicCode.Id} is missing.", - CreatedAt = DateTime.UtcNow - }); - continue; - } - BaseCalc calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", - CreatedAt = DateTime.UtcNow - }); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", - CreatedAt = DateTime.UtcNow - }); - } + newRecords.Add(calc.CalculateT3(newRecords)); } catch (Exception e) { - logsController.AddEntry(new LogEntry + _logsController.AddEntry(new LogEntry { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", CreatedAt = DateTime.UtcNow }); } } + catch (Exception e) + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", + CreatedAt = DateTime.UtcNow + }); + } } - - - if (isNew) - { - db.Layers.Add(processedLayer); - } - else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); } + + + if (isNew) + { + _db.Layers.Add(processedLayer); + } + else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); } -} +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs b/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs index 96d77f4..35f02e2 100644 --- a/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs @@ -1,226 +1,195 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; using Microsoft.EntityFrameworkCore; using WebAPI.Calculator; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors -{ - public class T3MultiSourceYearSummaryProcessor - { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; - private readonly LogsController logsController; +namespace WebAPI.dataProcessors; - public T3MultiSourceYearSummaryProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller, - LogsController _logsController) +public class T3MultiSourceYearSummaryProcessor +{ + private readonly AppDbContext _db; + private readonly LayersController _controller; + private readonly LogsController _logsController; + + public T3MultiSourceYearSummaryProcessor( + AppDbContext db, + LayersController controller, + LogsController logsController) + { + _db = db; + _controller = controller; + _logsController = logsController; + } + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (!sources!.Any()) { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - logsController = _logsController; + throw new Exception("Source record not found"); } - public void process(Layer processWorker) + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) { - int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - List? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count() == 0) + isNew = true; + processedLayer = new Layer { - throw new Exception("Source record not found"); - } - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + } - List newRecords = new List(); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; - List dataSources = new List(); - foreach (Record source in sources!) - { - Layer? dataSource = db.Layers.Where(x => - x.Type == LayerType.processed && - !x.IsDeleted && - x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3") - ) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource); - } - } - if (dataSources.Count == 0) - { - throw new Exception($"DataSources are empty"); - } + var newRecords = new List(); - List dataSourcesValidation = new List(); - for (int i = 1; i < 13; i++) - { - Layer? dataSource = db.Layers.Where(x => - x.Type == LayerType.processed - && !x.IsDeleted - && x.Name!=null && x.Name.Contains($"{year}/{i}-AA-T3")) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - dataSourcesValidation.Add(dataSource!); - } - } - if (dataSources.Count == 0) - { - throw new Exception($"DataSourcesValidation are empty"); - } - - - List allRecords = dataSources - .SelectMany(x => x.Records!).ToList(); - List baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")) + .Include(x => x.Records) + .FirstOrDefault()) + .OfType() + .ToList(); + if (dataSources.Count == 0) + { + throw new Exception("DataSources are empty"); + } + + if (dataSources.Count == 0) + { + throw new Exception("DataSourcesValidation are empty"); + } + + var allRecords = dataSources + .SelectMany(x => x.Records!).ToList(); + var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); - foreach (string baseCode in baseCodess) + foreach (var baseCode in baseCodes) + { + + var codeRecords = allRecords.Where(x => + x.Code![1..] == baseCode) + .ToList(); + var codeRecordsValidation = allRecords.Where(x => + x.Code![1..] == baseCode) + .ToList(); + var processedRecord = new Record { + Id = Guid.NewGuid(), + Code = $"9{baseCode}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + var validationRecord = new Record(); + for (var i = 1; i < 33; i++) + { + ProcessHelper.SetValue(processedRecord, i, + codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - List codeRecords = allRecords.Where(x => - x.Code!.Substring(1) == baseCode) - .ToList(); - List codeRecordsValidation = allRecords.Where(x => - x.Code!.Substring(1) == baseCode) - .ToList(); - Record processedRecord = new Record + ProcessHelper.SetValue(validationRecord, i, + codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); + + if ( + double.Abs((double)(ProcessHelper.GetValue(processedRecord,i) - + ProcessHelper.GetValue(validationRecord, i))!) > 0.01) { - Id = Guid.NewGuid(), - Code = $"9{baseCode}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - Record validationRecord = new Record(); - for (var i = 1; i < 33; i++) + throw new Exception($"ValidationError: Code {baseCode}, " + + $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + + $"{ProcessHelper.GetValue(validationRecord, i)})"); + } + } + newRecords.Add(processedRecord); + } + + + + // Dynamic Codes + var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); + if (dynamicCodes != null && dynamicCodes.Any()) + { + foreach (var dynamicCode in dynamicCodes) + { + try { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - - ProcessHelper.SetValue(validationRecord, i, - codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); - - if ( - ProcessHelper.GetValue(processedRecord,i) != - ProcessHelper.GetValue(validationRecord, i)) + if (dynamicCode.Desc1 == null) { - throw new Exception($"ValidationError: Code {baseCode!}, " + - $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + - $"{ProcessHelper.GetValue(validationRecord, i)})"); + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula in Record {dynamicCode.Id} is missing.", + CreatedAt = DateTime.UtcNow + }); + continue; } - } - newRecords.Add(processedRecord); - } - - - - // Dynamic Codes - List? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); - if (dynamicCodes != null && dynamicCodes.Any()) - { - foreach (Record dynamicCode in dynamicCodes) - { + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", + CreatedAt = DateTime.UtcNow + }); + continue; + } + try { - if (dynamicCode.Desc1 == null) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula in Record {dynamicCode.Id} is missing.", - CreatedAt = DateTime.UtcNow - }); - continue; - } - BaseCalc calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", - CreatedAt = DateTime.UtcNow - }); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", - CreatedAt = DateTime.UtcNow - }); - } + newRecords.Add(calc.CalculateT3(newRecords)); } catch (Exception e) { - logsController.AddEntry(new LogEntry + _logsController.AddEntry(new LogEntry { - Title = $"{processWorker!.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", CreatedAt = DateTime.UtcNow }); } } + catch (Exception e) + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", + CreatedAt = DateTime.UtcNow + }); + } } - if (isNew) - { - db.Layers.Add(processedLayer); - } - else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); } + if (isNew) + { + _db.Layers.Add(processedLayer); + } + else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); } -} +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t3.SingleSource.processor.cs b/WebAPI/dataProcessors/t3.SingleSource.processor.cs index 13e64a1..197b854 100644 --- a/WebAPI/dataProcessors/t3.SingleSource.processor.cs +++ b/WebAPI/dataProcessors/t3.SingleSource.processor.cs @@ -1,166 +1,147 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; -using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using System; -using System.Globalization; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors -{ - public class T3SingleSourceProcessor - { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; +namespace WebAPI.dataProcessors; - public T3SingleSourceProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) +public class T3SingleSourceProcessor +{ + private readonly AppDbContext _db; + private readonly LayersController _controller; + + public T3SingleSourceProcessor( + AppDbContext db, + LayersController controller) + { + _db = db; + _controller = controller; + } + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; + if (sourceLayer == null) { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; + throw new Exception("SourceLayer record not found"); + } + var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer); + if (sourceImportWorker == null) + { + throw new Exception("SourceImportWorkerL layer not found"); + } + var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; + if (sourceLayer == null) + { + throw new Exception("Source record not found"); } - public void process(Layer processWorker) + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) { - int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) + isNew = true; + processedLayer = new Layer { - throw new Exception("SourceLayer record not found"); - } - Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer); - if (sourceImportWorker == null) - { - throw new Exception("SourceImportWorkerL layer not found"); - } - string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("Source record not found"); - } - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; - List newRecords = new List(); + var newRecords = new List(); - List dataSources = db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted) - .OrderBy(x => x.CreatedAt) - .ToList(); - if (dataSources.Count == 0) + var dataSources = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == sourceImportWorker.Id + && !x.IsDeleted) + .OrderBy(x => x.CreatedAt) + .ToList(); + if (dataSources.Count == 0) + { + throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); + } + + + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + foreach (var baseRecord in dataSources.Last().Records!) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + + var processedRecord = new Record { - throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); - } + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; - - List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (Record baseRecord in dataSources.Last()?.Records!) + var lastDayInMonth = DateTime.DaysInMonth(year, month); + //day 1 + var firstVal = codeRecords + .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0; + ProcessHelper.SetValue(processedRecord, 1, firstVal); + var previousValue = firstVal; + //days 2-29/30 + for (var i=2; i codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - - Record processedRecord = new Record + var dayVal = codeRecords + .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1; + if (dayVal == null) { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - int lastDayInMonth = DateTime.DaysInMonth(year, month); - double previousValue = 0; - //day 1 - double firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1 ?? 0; - ProcessHelper.SetValue(processedRecord, 1, firstVal); - previousValue = firstVal; - //days 2-29/30 - for (int i=2; i x.CreatedAt.Day == i && x.CreatedAt.Month == month) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1; - if (dayVal == null) - { - //TODO: missing day value? Should I log it? - ProcessHelper.SetValue(processedRecord, i, 0); - } else - { - double processedVal = (dayVal ?? 0) - previousValue; - ProcessHelper.SetValue(processedRecord, i, processedVal); - previousValue = dayVal ?? 0; - } - } - //last day - double? lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1; - - if (lastVal == null) - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); + ProcessHelper.SetValue(processedRecord, i, 0); } else { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue); + var processedVal = dayVal - previousValue; + ProcessHelper.SetValue(processedRecord, i, processedVal); + previousValue = (double)dayVal; } - - // copy last value - double? valueToCopy = codeRecords - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault()?.Value1; - ProcessHelper.SetValue(processedRecord, 32, valueToCopy); - - newRecords.Add(processedRecord); } - - if (isNew) + //last day + var lastVal = codeRecords + .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1; + + if (lastVal == null) { - db.Layers.Add(processedLayer); + ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); } else { - db.Layers.Update(processedLayer); + ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); + + // copy last value + var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; + ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + + newRecords.Add(processedRecord); } - + + if (isNew) + { + _db.Layers.Add(processedLayer); + } else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); } -} + +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs b/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs index 4d425dc..883f215 100644 --- a/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs +++ b/WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs @@ -1,115 +1,110 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; using Microsoft.EntityFrameworkCore; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors +namespace WebAPI.dataProcessors; + +public class T3SourceYearSummaryProcessor { - public class T3SourceYearSummaryProcessor + private readonly AppDbContext _db; + private readonly LayersController _controller; + + public T3SourceYearSummaryProcessor( + AppDbContext db, + LayersController controller + ) { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; - - public T3SourceYearSummaryProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller - ) - { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - } - - public void process(Layer processWorker) - { - string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; - string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - List newRecords = new List(); - - List dataSources = new List(); - for (int i=1; i<13; i++) - { - Layer? dataSource = db.Layers.Where(x => - x.Type == LayerType.processed - && !x.IsDeleted - && x.Name!=null && x.Name.Contains($"{year}/{i}-{source}-T3")) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource!); - } - } - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - List allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (Record baseRecord in dataSources.Last()?.Records!) - { - List codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - Record processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - newRecords.Add(processedRecord); - } - - if (isNew) - { - db.Layers.Add(processedLayer); - } else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); - } + _db = db; + _controller = controller; } -} + + public void Process(Layer processWorker) + { + var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; + var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; + if (source == null) + { + throw new Exception("Source record not found"); + } + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + var newRecords = new List(); + + var dataSources = new List(); + for (var i=1; i<13; i++) + { + var i1 = i; + var dataSource = _db.Layers.Where(x => + x.Type == LayerType.Processed + && !x.IsDeleted + && x.Name!=null && x.Name.Contains($"{year}/{i1}-{source}-T3")) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSource != null) + { + dataSources.Add(dataSource); + } + } + + if (dataSources.Count == 0) + { + throw new Exception("DataSources are empty"); + } + + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + foreach (var baseRecord in dataSources.Last().Records!) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + for (var i = 1; i < 33; i++) + { + ProcessHelper.SetValue(processedRecord, i, + codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + } + newRecords.Add(processedRecord); + } + + if (isNew) + { + _db.Layers.Add(processedLayer); + } else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); + } +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t4.SingleSource.processor.cs b/WebAPI/dataProcessors/t4.SingleSource.processor.cs index 05e9510..d1b9248 100644 --- a/WebAPI/dataProcessors/t4.SingleSource.processor.cs +++ b/WebAPI/dataProcessors/t4.SingleSource.processor.cs @@ -1,116 +1,101 @@ -using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; -using Microsoft.AspNetCore.Mvc; -using Microsoft.EntityFrameworkCore; -using System; -using System.Globalization; +using Microsoft.EntityFrameworkCore; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors +namespace WebAPI.dataProcessors; + +public class T4SingleSourceProcessor { - public class T4SingleSourceProcessor + private readonly AppDbContext _db; + private readonly LayersController _controller; + + public T4SingleSourceProcessor( + AppDbContext db, + LayersController controller) { - private readonly AppDbContext db; - private readonly SpreadsheetsResource.ValuesResource googleSheetValues; - private readonly LayersController controller; - - public T4SingleSourceProcessor( - AppDbContext _db, - SpreadsheetsResource.ValuesResource _googleSheetValues, - LayersController _controller) - { - db = _db; - googleSheetValues = _googleSheetValues; - controller = _controller; - } - - public void process(Layer processWorker) - { - int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("SourceLayer record not found"); - } - Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer); - if (sourceImportWorker == null) - { - throw new Exception("SourceImportWorkerL layer not found"); - } - string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("Source record not found"); - } - - Layer? processedLayer = db.Layers - .Where(x => x.ParentId == processWorker!.Id) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - bool isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker!.Id, - Number = db.Layers.Count() + 1, - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T4"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - processedLayer.Sources = new List(); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - List newRecords = new List(); - - Layer? dataSource = db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - if (dataSource == null) - { - throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); - } - - foreach (Record record in dataSource.Records!) - { - Record processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = record.Code, - Desc1 = record.Desc1, - Value1 = record.Value1, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - newRecords.Add(processedRecord); - } - - if (isNew) - { - db.Layers.Add(processedLayer); - } else - { - db.Layers.Update(processedLayer); - } - controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - db.SaveChanges(); - } - + _db = db; + _controller = controller; } -} + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); + var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; + if (sourceLayer == null) + { + throw new Exception("SourceLayer record not found"); + } + var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer); + if (sourceImportWorker == null) + { + throw new Exception("SourceImportWorkerL layer not found"); + } + var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; + if (sourceLayer == null) + { + throw new Exception("Source record not found"); + } + + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T4"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; + processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + + var dataSource = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == sourceImportWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (dataSource == null) + { + throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); + } + + var newRecords = dataSource.Records!.Select(record => new Record + { + Id = Guid.NewGuid(), + Code = record.Code, + Desc1 = record.Desc1, + Value1 = record.Value1, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }) + .ToList(); + + if (isNew) + { + _db.Layers.Add(processedLayer); + } else + { + _db.Layers.Update(processedLayer); + } + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); + } + +} \ No newline at end of file diff --git a/WebAPI/dataProcessors/t4.r2.processor.cs b/WebAPI/dataProcessors/t4.r2.processor.cs index c7ff298..f651260 100644 --- a/WebAPI/dataProcessors/t4.r2.processor.cs +++ b/WebAPI/dataProcessors/t4.r2.processor.cs @@ -1,189 +1,176 @@ using DiunaBIWebAPI.dataProcessors; -using Google.Apis.Sheets.v4; using Microsoft.EntityFrameworkCore; using WebAPI.Controllers; using WebAPI.Models; -namespace WebAPI.dataProcessors -{ - public class T4R2Processor - { - private readonly AppDbContext _db; - private readonly LayersController _controller; - private readonly LogsController _logsController; +namespace WebAPI.dataProcessors; - public T4R2Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - LayersController controller, - LogsController logsController) +public class T4R2Processor +{ + private readonly AppDbContext _db; + private readonly LayersController _controller; + private readonly LogsController _logsController; + + public T4R2Processor( + AppDbContext db, + LayersController controller, + LogsController logsController) + { + _db = db; + _controller = controller; + _logsController = logsController; + } + + public void Process(Layer processWorker) + { + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); + if (!sources!.Any()) { - _db = db; - _controller = controller; - _logsController = logsController; + throw new Exception("Source record not found"); } - public void Process(Layer processWorker) + var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; + if (layerName == null) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (!sources!.Any()) + throw new Exception("LayerName record not found"); + } + + + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id + && !x.IsDeleted) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + var isNew = false; + if (processedLayer == null) + { + isNew = true; + processedLayer = new Layer { - throw new Exception("Source record not found"); - } - - var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; - if (layerName == null) - { - throw new Exception("LayerName record not found"); - } - - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Source = "", - Type = LayerType.processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.Sources = new List(); + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1 + }; + processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; + processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + } + + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; - var newRecords = new List(); + var newRecords = new List(); - foreach (var source in sources!) + foreach (var source in sources!) + { + var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") + ?.Desc1; + var sourceCodes = new List(); + if (rawSourceCodes != null) { - var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") - ?.Desc1; - var sourceCodes = new List(); - if (rawSourceCodes != null) - { - sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); - } - - for (var month = 1; month <= DateTime.UtcNow.Month; month++) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.processed && - !x.IsDeleted && - x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T") - ) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSource != null) - { - List news = dataSource.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code + month.ToString("D2"), - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, - Desc1 = x.Desc1 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", - CreatedAt = DateTime.UtcNow - }); - } - } - - // year summery - var dataSourceSum = _db.Layers.Where(x => - x.Type == LayerType.processed && - !x.IsDeleted && - x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") - ) - .Include(x => x.Records) - .FirstOrDefault(); - if (dataSourceSum != null) - { - var news = dataSourceSum.Records! - .Where(x => - { - if (sourceCodes.Count > 0) - { - return sourceCodes.Contains(int.Parse(x.Code!)); - } - - return true; // get all - }) - .Select(x => - { - Record newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code + "13", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.warning, - LogType = LogType.process, - Message = $"Data source {year}/13-{source.Desc1}-T3 not found", - CreatedAt = DateTime.UtcNow - }); - } + sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); } - - if (isNew) + + for (var month = 1; month <= DateTime.UtcNow.Month; month++) { - _db.Layers.Add(processedLayer); + var monthCopy = month; + var dataSource = _db.Layers.Where(x => + x.Type == LayerType.Processed && + !x.IsDeleted && + x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T") + ) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSource != null) + { + var news = dataSource.Records! + .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) + .Select(x => + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = x.Code + month.ToString("D2"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, + Desc1 = x.Desc1 + }; + return newRecord; + } + ).ToList(); + newRecords.AddRange(news); + } + else + { + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", + CreatedAt = DateTime.UtcNow + }); + } + } + + // year summery + var dataSourceSum = _db.Layers.Where(x => + x.Type == LayerType.Processed && + !x.IsDeleted && + x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") + ) + .Include(x => x.Records) + .FirstOrDefault(); + if (dataSourceSum != null) + { + var news = dataSourceSum.Records! + .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) + .Select(x => + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = x.Code + "13", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }; + return newRecord; + } + ).ToList(); + newRecords.AddRange(news); } else { - _db.Layers.Update(processedLayer); + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Process, + Message = $"Data source {year}/13-{source.Desc1}-T3 not found", + CreatedAt = DateTime.UtcNow + }); } - - _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - _db.SaveChanges(); } + + if (isNew) + { + _db.Layers.Add(processedLayer); + } + else + { + _db.Layers.Update(processedLayer); + } + + _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + _db.SaveChanges(); } } \ No newline at end of file