mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code refactor finished

Browse Source
This commit is contained in:
Michał Zieliski
2024-06-18 22:02:19 +02:00
parent 7e8406dc9b
commit 763b4a8c4d
19 changed files with 1105 additions and 1228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
WebAPI/Controllers/AdminController.cs
View File

@@ -85,8 +85,8 @@ public class AdminController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "Backup success", Title = "Backup success",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.backup, LogType = LogType.Backup,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Ok(); return Ok();
@@ -96,8 +96,8 @@ public class AdminController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "Backup error", Title = "Backup error",
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.backup, LogType = LogType.Backup,
Message = e.ToString(), Message = e.ToString(),
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });

120
WebAPI/Controllers/LayersController.cs
View File

@@ -90,8 +90,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Unauthorized request - wrong apiKey ({number})", Title = $"Unauthorized request - wrong apiKey ({number})",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Unauthorized(); return Unauthorized();
@@ -105,8 +105,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Unauthorized request - no authorization header ({number})", Title = $"Unauthorized request - no authorization header ({number})",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Unauthorized(); return Unauthorized();
@@ -118,8 +118,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Unauthorized request - wrong auth header format ({number})", Title = $"Unauthorized request - wrong auth header format ({number})",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Unauthorized(); return Unauthorized();
@@ -133,8 +133,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Unauthorized request - bad credentials ({number})", Title = $"Unauthorized request - bad credentials ({number})",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Unauthorized(); return Unauthorized();
@@ -143,8 +143,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Sending data for layer {number}", Title = $"Sending data for layer {number}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -157,8 +157,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = e.ToString(), Title = e.ToString(),
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.powerBI, LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return BadRequest(e.ToString()); return BadRequest(e.ToString());
@@ -213,8 +213,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "No Layers to import.", Title = "No Layers to import.",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.import, LogType = LogType.Import,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
return Ok(); return Ok();
@@ -235,8 +235,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{importWorker.Name}, {importWorker.Id}", Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.import, LogType = LogType.Import,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -267,8 +267,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{importWorker.Name}, {importWorker.Id}", Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.import, LogType = LogType.Import,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -282,8 +282,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{importWorker.Name}, {importWorker.Id}", Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.import, LogType = LogType.Import,
Message = "Success (reimported)", Message = "Success (reimported)",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -293,8 +293,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{importWorker.Name}, {importWorker.Id}", Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.import, LogType = LogType.Import,
Message = "importLayer records are up of date. Not processed.", Message = "importLayer records are up of date. Not processed.",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -306,8 +306,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{importWorker.Name}, {importWorker.Id}", Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.import, LogType = LogType.Import,
Message = e.ToString(), Message = e.ToString(),
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -321,8 +321,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "Process error", Title = "Process error",
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.import, LogType = LogType.Import,
Message = e.ToString(), Message = e.ToString(),
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -363,8 +363,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"Processing: {type}", Title = $"Processing: {type}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -383,8 +383,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "No Layers to process.", Title = "No Layers to process.",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
} }
@@ -400,8 +400,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.process, LogType = LogType.Process,
Message = e.ToString(), Message = e.ToString(),
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -413,8 +413,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = "Process error", Title = "Process error",
Type = LogEntryType.error, Type = LogEntryType.Error,
LogType = LogType.process, LogType = LogType.Process,
Message = e.ToString(), Message = e.ToString(),
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -445,14 +445,14 @@ public class LayersController : Controller
case "T3-SourceYearSummary": case "T3-SourceYearSummary":
{ {
var processor = var processor =
new T3SourceYearSummaryProcessor(_db, _googleSheetValues, this); new T3SourceYearSummaryProcessor(_db, this);
processor.process(processWorker); processor.Process(processWorker);
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -461,14 +461,14 @@ public class LayersController : Controller
case "T3-MultiSourceYearSummary": case "T3-MultiSourceYearSummary":
{ {
var processor = var processor =
new T3MultiSourceYearSummaryProcessor(_db, _googleSheetValues, this, _logsController); new T3MultiSourceYearSummaryProcessor(_db, this, _logsController);
processor.process(processWorker); processor.Process(processWorker);
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -477,14 +477,14 @@ public class LayersController : Controller
case "T3-MultiSourceCopySelectedCodesYearSummary": case "T3-MultiSourceCopySelectedCodesYearSummary":
{ {
var processor = var processor =
new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, _googleSheetValues, this); new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this);
processor.process(processWorker); processor.Process(processWorker);
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -498,8 +498,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -507,14 +507,14 @@ public class LayersController : Controller
} }
case "T4-R2": case "T4-R2":
{ {
var processor = new T4R2Processor(_db, _googleSheetValues, this, _logsController); var processor = new T4R2Processor(_db, this, _logsController);
processor.Process(processWorker); processor.Process(processWorker);
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -531,28 +531,28 @@ public class LayersController : Controller
switch (processType!) switch (processType!)
{ {
case "T3-SingleSource": case "T3-SingleSource":
var t3SingleSource = new T3SingleSourceProcessor(_db, _googleSheetValues, this); var t3SingleSource = new T3SingleSourceProcessor(_db, this);
{ {
t3SingleSource.process(processWorker); t3SingleSource.Process(processWorker);
break; break;
} }
case "T4-SingleSource": case "T4-SingleSource":
{ {
var t4SingleSource = new T4SingleSourceProcessor(_db, _googleSheetValues, this); var t4SingleSource = new T4SingleSourceProcessor(_db, this);
t4SingleSource.process(processWorker); t4SingleSource.Process(processWorker);
break; break;
} }
case "T3-MultiSourceSummary": case "T3-MultiSourceSummary":
{ {
var t3MultiSourceSummary = var t3MultiSourceSummary =
new T3MultiSourceSummaryProcessor(_db, _googleSheetValues, this, _logsController); new T3MultiSourceSummaryProcessor(_db, this, _logsController);
t3MultiSourceSummary.process(processWorker); t3MultiSourceSummary.Process(processWorker);
break; break;
} }
case "T3-MultiSourceCopySelectedCodes": case "T3-MultiSourceCopySelectedCodes":
{ {
var t3MultiSourceCopySelectedCode = var t3MultiSourceCopySelectedCode =
new T3MultiSourceCopySelectedCodesProcessor(_db, _googleSheetValues, this); new T3MultiSourceCopySelectedCodesProcessor(_db, this);
t3MultiSourceCopySelectedCode.Process(processWorker); t3MultiSourceCopySelectedCode.Process(processWorker);
break; break;
} }
@@ -561,8 +561,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info, Type = LogEntryType.Info,
LogType = LogType.process, LogType = LogType.Process,
Message = "Success", Message = "Success",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });

8
WebAPI/Controllers/LogsController.cs
View File

@@ -28,10 +28,10 @@ public class LogsController : Controller
var type = entry.LogType switch var type = entry.LogType switch
{ {
LogType.import => "Import", LogType.Import => "Import",
LogType.backup => "Backup", LogType.Backup => "Backup",
LogType.process => "Process", LogType.Process => "Process",
LogType.powerBI => "PowerBIAccess", LogType.PowerBi => "PowerBIAccess",
_ => "Other" _ => "Other"
}; };
var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute(); var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();

6
WebAPI/Exports/googleSheet.export.cs
View File

@@ -32,19 +32,19 @@ public class GoogleSheetExport
switch (layer.Type) switch (layer.Type)
{ {
case LayerType.import: case LayerType.Import:
{ {
data.Add(new List<object> { "Code", "Value1" }); data.Add(new List<object> { "Code", "Value1" });
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Value1! })); data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Value1! }));
break; break;
} }
case LayerType.administration: case LayerType.Administration:
{ {
data.Add(new List<object> { "Code", "Desc1"}); data.Add(new List<object> { "Code", "Desc1"});
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Desc1! })); data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Desc1! }));
break; break;
} }
case LayerType.processed: case LayerType.Processed:
{ {
data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3", data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3",
"Value5", "Value6", "Value7", "Value8", "Value9", "Value10", "Value5", "Value6", "Value7", "Value8", "Value9", "Value10",

42
WebAPI/Models/Layer.cs
View File

@@ -1,44 +1,42 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models namespace WebAPI.Models;
public enum LayerType
{
Import,
Processed,
Administration
}
public class Layer
{ {
public enum LayerType
{
import,
processed,
administration,
}
public class Layer
{
#region Properties #region Properties
[Key] [Key]
public Guid Id { get; set; } public Guid Id { get; init; }
[Required] [Required]
public int Number { get; set; } public int Number { get; init; }
[Required]
public string? Source { get; set; }
[Required] [Required]
[MaxLength(50)]
public string? Name { get; set; } public string? Name { get; set; }
[Required] [Required]
public LayerType Type { get; set; } public LayerType Type { get; init; }
[Required] [Required]
public DateTime CreatedAt { get; set; } public DateTime CreatedAt { get; set; }
[Required] [Required]
public DateTime ModifiedAt { get; set; } public DateTime ModifiedAt { get; set; }
[Required] [Required]
public bool IsDeleted { get; set; } public bool IsDeleted { get; init; }
#endregion #endregion
#region Relations #region Relations
public ICollection<Record>? Records { get; set; } public ICollection<Record>? Records { get; init; }
public ICollection<ProcessSource>? Sources { get; set; }
[Required] [Required]
public Guid CreatedById { get; set; } public Guid CreatedById { get; set; }
public User? CreatedBy { get; set; } public User? CreatedBy { get; init; }
[Required] [Required]
public Guid ModifiedById { get; set; } public Guid ModifiedById { get; set; }
public User? ModifiedBy { get; set; } public User? ModifiedBy { get; init; }
public Guid? ParentId { get; set; } public Guid? ParentId { get; init; }
public Layer? Parent { get; set; } public Layer? Parent { get; init; }
#endregion #endregion
}
} }

41
WebAPI/Models/LogEntry.cs
View File

@@ -1,25 +1,22 @@
using System.ComponentModel.DataAnnotations; namespace WebAPI.Models;
namespace WebAPI.Models public enum LogEntryType
{ {
public enum LogEntryType Info,
{ Warning,
info, Error
warning, }
error, public enum LogType {
} Import,
public enum LogType { Backup,
import, Process,
backup, PowerBi
process, }
powerBI public class LogEntry
} {
public class LogEntry public LogType LogType { get; init; }
{ public LogEntryType Type { get; init; }
public LogType LogType { get; set; } public string? Message { get; init; }
public LogEntryType Type { get; set; } public string? Title {get; init;}
public string? Message { get; set; } public DateTime CreatedAt { get; init; }
public string? Title {get; set;}
public DateTime CreatedAt { get; set; }
}
} }

13
WebAPI/Models/ProcessSource.cs
View File

@@ -1,15 +1,14 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models namespace WebAPI.Models;
public class ProcessSource
{ {
public class ProcessSource
{
#region Relations #region Relations
[Required] [Required]
public Guid LayerId { get; set; } public Guid LayerId { get; init; }
[Required] [Required]
public Guid SourceId { get; set; } public Guid SourceId { get; init; }
public Layer? Source { get; set; } public Layer? Source { get; init; }
#endregion #endregion
}
} }

14
WebAPI/Models/Record.cs
View File

@@ -6,10 +6,10 @@ public class Record
{ {
#region Properties #region Properties
[Key] [Key]
public Guid Id { get; set; } public Guid Id { get; init; }
[Required] [Required]
[StringLength(50)] [StringLength(50)]
public string? Code { get; set; } public string? Code { get; init; }
public double? Value1 { get; set; } public double? Value1 { get; set; }
public double? Value2 { get; set; } public double? Value2 { get; set; }
public double? Value3 { get; set; } public double? Value3 { get; set; }
@@ -44,15 +44,15 @@ public class Record
public double? Value32 { get; set; } public double? Value32 { get; set; }
//Description fields //Description fields
[StringLength(50)] [StringLength(50)]
public string? Desc1 { get; set; } public string? Desc1 { get; init; }
[StringLength(50)] [StringLength(50)]
public string? Desc2 { get; set; } public string? Desc2 { get; init; }
[StringLength(50)] [StringLength(50)]
public string? Desc3 { get; set; } public string? Desc3 { get; init; }
[StringLength(50)] [StringLength(50)]
public string? Desc4 { get; set; } public string? Desc4 { get; init; }
[StringLength(50)] [StringLength(50)]
public string? Desc5 { get; set; } public string? Desc5 { get; init; }
[StringLength(50)] [StringLength(50)]
public DateTime CreatedAt { get; set; } public DateTime CreatedAt { get; set; }
public DateTime ModifiedAt { get; set; } public DateTime ModifiedAt { get; set; }

3
WebAPI/dataImporters/morska.fk2.importer.cs
View File

@@ -80,10 +80,9 @@ public class MorskaFk2Importer
} }
var layer = new Layer var layer = new Layer
{ {
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.import, Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,

3
WebAPI/dataImporters/morska.importer.cs
View File

@@ -80,10 +80,9 @@ public class MorskaImporter
} }
var layer = new Layer var layer = new Layer
{ {
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.import, Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,

17
WebAPI/dataProcessors/t1.r1.processor.cs
View File

@@ -55,8 +55,7 @@ public class T1R1Processor
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1
}; };
@@ -66,7 +65,7 @@ public class T1R1Processor
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
@@ -78,7 +77,7 @@ public class T1R1Processor
{ {
var monthCopy = month; var monthCopy = month;
var dataSource = _db.Layers.Where(x => var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T3") x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T3")
) )
@@ -93,8 +92,8 @@ public class T1R1Processor
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -102,7 +101,7 @@ public class T1R1Processor
} }
// year summary // year summary
var dataSourceSum = _db.Layers.Where(x => var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3") x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
) )
@@ -117,8 +116,8 @@ public class T1R1Processor
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found", Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });

53
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs
View File

@@ -1,32 +1,27 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3MultiSourceCopySelectedCodesProcessor
{ {
public class T3MultiSourceCopySelectedCodesProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3MultiSourceCopySelectedCodesProcessor( public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller)
LayersController _controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller;
} }
public void Process(Layer processWorker) public void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (!sources!.Any()) if (!sources!.Any())
{ {
@@ -40,8 +35,8 @@ namespace WebAPI.dataProcessors
var codesList = ProcessHelper.ParseCodes(codes); var codesList = ProcessHelper.ParseCodes(codes);
var processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
@@ -53,10 +48,9 @@ namespace WebAPI.dataProcessors
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -64,18 +58,18 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
var dataSources = sources!.Select(source => db.Layers.Where(x => x.Type == LayerType.processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")) var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault()) .FirstOrDefault())
.OfType<Layer>() .OfType<Layer>()
.ToList(); .ToList();
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception($"DataSources are empty"); throw new Exception("DataSources are empty");
} }
@@ -100,14 +94,13 @@ namespace WebAPI.dataProcessors
.ToList(); .ToList();
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} }
else else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
} }
} }

73
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs
View File

@@ -1,49 +1,43 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor
{ {
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3MultiSourceCopySelectedCodesYearSummaryProcessor( public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller)
LayersController _controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -52,25 +46,25 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>(); var dataSources = new List<Layer>();
for (int i = 1; i < 13; i++) for (var i = 1; i < 13; i++)
{ {
Layer? dataSource = db.Layers.Where(x => var i1 = i;
x.Type == LayerType.processed var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed
&& !x.IsDeleted && !x.IsDeleted
&& x.Name != null && x.Name.Contains($"{year}/{i}-AB-T3")) && x.Name != null && x.Name.Contains($"{year}/{i1}-AB-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
dataSources.Add(dataSource!); dataSources.Add(dataSource);
} }
} }
@@ -79,12 +73,12 @@ namespace WebAPI.dataProcessors
throw new Exception("DataSources are empty"); throw new Exception("DataSources are empty");
} }
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!) foreach (var baseRecord in dataSources.Last().Records!)
{ {
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = baseRecord.Code, Code = baseRecord.Code,
@@ -102,15 +96,14 @@ namespace WebAPI.dataProcessors
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} }
else else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
} }
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges(); _db.SaveChanges();
}
} }
} }

135
WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
View File

@@ -1,58 +1,53 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Calculator; using WebAPI.Calculator;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3MultiSourceSummaryProcessor
{ {
public class T3MultiSourceSummaryProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db; private readonly LogsController _logsController;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
public T3MultiSourceSummaryProcessor( public T3MultiSourceSummaryProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller,
LayersController _controller, LogsController logsController)
LogsController _logsController)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller; _logsController = logsController;
logsController = _logsController;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0) if (!sources!.Any())
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -60,44 +55,33 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>(); var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3"))
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name!=null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
)
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault(); .FirstOrDefault())
if (dataSource != null) .OfType<Layer>()
{ .ToList();
dataSources.Add(dataSource);
}
}
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception($"DataSources are empty"); throw new Exception("DataSources are empty");
} }
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
List<string> baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (string baseCode in baseCodess) foreach (var baseCode in baseCodes)
{ {
List<Record> codeRecords = allRecords.Where(x => var codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode) x.Code![1..] == baseCode)
.ToList(); .ToList();
Record processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"9{baseCode}", Code = $"9{baseCode}",
@@ -113,33 +97,33 @@ namespace WebAPI.dataProcessors
} }
// Dynamic Codes // Dynamic Codes
List<Record>? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
if (dynamicCodes != null && dynamicCodes.Any()) if (dynamicCodes != null && dynamicCodes.Any())
{ {
foreach (Record dynamicCode in dynamicCodes) foreach (var dynamicCode in dynamicCodes)
{ {
try try
{ {
if (dynamicCode.Desc1 == null) if (dynamicCode.Desc1 == null)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.", Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
continue; continue;
} }
BaseCalc calc = new BaseCalc(dynamicCode.Desc1); var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect()) if (!calc.IsFormulaCorrect())
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -152,11 +136,11 @@ namespace WebAPI.dataProcessors
} }
catch (Exception e) catch (Exception e)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -164,11 +148,11 @@ namespace WebAPI.dataProcessors
} }
catch (Exception e) catch (Exception e)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -179,14 +163,13 @@ namespace WebAPI.dataProcessors
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} }
else else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
} }
} }

161
WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs
View File

@@ -1,57 +1,52 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Calculator; using WebAPI.Calculator;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3MultiSourceYearSummaryProcessor
{ {
public class T3MultiSourceYearSummaryProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db; private readonly LogsController _logsController;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
public T3MultiSourceYearSummaryProcessor( public T3MultiSourceYearSummaryProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller,
LayersController _controller, LogsController logsController)
LogsController _logsController)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller; _logsController = logsController;
logsController = _logsController;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0) if (!sources!.Any())
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -59,73 +54,48 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>(); var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3"))
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault(); .FirstOrDefault())
if (dataSource != null) .OfType<Layer>()
{ .ToList();
dataSources.Add(dataSource);
}
}
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception($"DataSources are empty"); throw new Exception("DataSources are empty");
} }
List<Layer> dataSourcesValidation = new List<Layer>();
for (int i = 1; i < 13; i++)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed
&& !x.IsDeleted
&& x.Name!=null && x.Name.Contains($"{year}/{i}-AA-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSourcesValidation.Add(dataSource!);
}
}
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception($"DataSourcesValidation are empty"); throw new Exception("DataSourcesValidation are empty");
} }
var allRecords = dataSources
List<Record> allRecords = dataSources
.SelectMany(x => x.Records!).ToList(); .SelectMany(x => x.Records!).ToList();
List<string> baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (string baseCode in baseCodess) foreach (var baseCode in baseCodes)
{ {
List<Record> codeRecords = allRecords.Where(x => var codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode) x.Code![1..] == baseCode)
.ToList(); .ToList();
List<Record> codeRecordsValidation = allRecords.Where(x => var codeRecordsValidation = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode) x.Code![1..] == baseCode)
.ToList(); .ToList();
Record processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"9{baseCode}", Code = $"9{baseCode}",
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
Record validationRecord = new Record(); var validationRecord = new Record();
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.SetValue(processedRecord, i, ProcessHelper.SetValue(processedRecord, i,
@@ -135,10 +105,10 @@ namespace WebAPI.dataProcessors
codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i)));
if ( if (
ProcessHelper.GetValue(processedRecord,i) != double.Abs((double)(ProcessHelper.GetValue(processedRecord,i) -
ProcessHelper.GetValue(validationRecord, i)) ProcessHelper.GetValue(validationRecord, i))!) > 0.01)
{ {
throw new Exception($"ValidationError: Code {baseCode!}, " + throw new Exception($"ValidationError: Code {baseCode}, " +
$"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " +
$"{ProcessHelper.GetValue(validationRecord, i)})"); $"{ProcessHelper.GetValue(validationRecord, i)})");
} }
@@ -149,33 +119,33 @@ namespace WebAPI.dataProcessors
// Dynamic Codes // Dynamic Codes
List<Record>? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList(); var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
if (dynamicCodes != null && dynamicCodes.Any()) if (dynamicCodes != null && dynamicCodes.Any())
{ {
foreach (Record dynamicCode in dynamicCodes) foreach (var dynamicCode in dynamicCodes)
{ {
try try
{ {
if (dynamicCode.Desc1 == null) if (dynamicCode.Desc1 == null)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.", Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
continue; continue;
} }
BaseCalc calc = new BaseCalc(dynamicCode.Desc1); var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect()) if (!calc.IsFormulaCorrect())
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -188,11 +158,11 @@ namespace WebAPI.dataProcessors
} }
catch (Exception e) catch (Exception e)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -200,11 +170,11 @@ namespace WebAPI.dataProcessors
} }
catch (Exception e) catch (Exception e)
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -213,14 +183,13 @@ namespace WebAPI.dataProcessors
} }
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} }
else else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
} }
} }

109
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -1,66 +1,58 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3SingleSourceProcessor
{ {
public class T3SingleSourceProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3SingleSourceProcessor( public T3SingleSourceProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller)
LayersController _controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null) if (sourceLayer == null)
{ {
throw new Exception("SourceLayer record not found"); throw new Exception("SourceLayer record not found");
} }
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer); var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
if (sourceImportWorker == null) if (sourceImportWorker == null)
{ {
throw new Exception("SourceImportWorkerL layer not found"); throw new Exception("SourceImportWorkerL layer not found");
} }
string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null) if (sourceLayer == null)
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id) .Where(x => x.ParentId == processWorker.Id)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -68,14 +60,14 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
List<Layer> dataSources = db.Layers var dataSources = _db.Layers
.Include(x => x.Records) .Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id .Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
@@ -87,13 +79,13 @@ namespace WebAPI.dataProcessors
} }
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!) foreach (var baseRecord in dataSources.Last().Records!)
{ {
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = baseRecord.Code, Code = baseRecord.Code,
@@ -101,51 +93,41 @@ namespace WebAPI.dataProcessors
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
int lastDayInMonth = DateTime.DaysInMonth(year, month); var lastDayInMonth = DateTime.DaysInMonth(year, month);
double previousValue = 0;
//day 1 //day 1
double firstVal = codeRecords var firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)) .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
ProcessHelper.SetValue(processedRecord, 1, firstVal); ProcessHelper.SetValue(processedRecord, 1, firstVal);
previousValue = firstVal; var previousValue = firstVal;
//days 2-29/30 //days 2-29/30
for (int i=2; i<lastDayInMonth; i++) for (var i=2; i<lastDayInMonth; i++)
{ {
double? dayVal = codeRecords var dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month) .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1;
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (dayVal == null) if (dayVal == null)
{ {
//TODO: missing day value? Should I log it?
ProcessHelper.SetValue(processedRecord, i, 0); ProcessHelper.SetValue(processedRecord, i, 0);
} else } else
{ {
double processedVal = (dayVal ?? 0) - previousValue; var processedVal = dayVal - previousValue;
ProcessHelper.SetValue(processedRecord, i, processedVal); ProcessHelper.SetValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0; previousValue = (double)dayVal;
} }
} }
//last day //last day
double? lastVal = codeRecords var lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)) .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1;
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
if (lastVal == null) if (lastVal == null)
{ {
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
} else } else
{ {
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue); ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
} }
// copy last value // copy last value
double? valueToCopy = codeRecords var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
ProcessHelper.SetValue(processedRecord, 32, valueToCopy); ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord); newRecords.Add(processedRecord);
@@ -153,14 +135,13 @@ namespace WebAPI.dataProcessors
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} else } else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
} }
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges(); _db.SaveChanges();
} }
}
} }

75
WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
View File

@@ -1,53 +1,48 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T3SourceYearSummaryProcessor
{ {
public class T3SourceYearSummaryProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T3SourceYearSummaryProcessor( public T3SourceYearSummaryProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller
LayersController _controller
) )
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null) if (source == null)
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -55,24 +50,25 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>(); var dataSources = new List<Layer>();
for (int i=1; i<13; i++) for (var i=1; i<13; i++)
{ {
Layer? dataSource = db.Layers.Where(x => var i1 = i;
x.Type == LayerType.processed var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed
&& !x.IsDeleted && !x.IsDeleted
&& x.Name!=null && x.Name.Contains($"{year}/{i}-{source}-T3")) && x.Name!=null && x.Name.Contains($"{year}/{i1}-{source}-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
dataSources.Add(dataSource!); dataSources.Add(dataSource);
} }
} }
@@ -81,12 +77,12 @@ namespace WebAPI.dataProcessors
throw new Exception("DataSources are empty"); throw new Exception("DataSources are empty");
} }
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!) foreach (var baseRecord in dataSources.Last().Records!)
{ {
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = baseRecord.Code, Code = baseRecord.Code,
@@ -103,13 +99,12 @@ namespace WebAPI.dataProcessors
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} else } else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
} }
} }

77
WebAPI/dataProcessors/t4.SingleSource.processor.cs
View File

@@ -1,66 +1,57 @@
using DiunaBIWebAPI.dataProcessors; using Microsoft.EntityFrameworkCore;
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T4SingleSourceProcessor
{ {
public class T4SingleSourceProcessor private readonly AppDbContext _db;
{ private readonly LayersController _controller;
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public T4SingleSourceProcessor( public T4SingleSourceProcessor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, LayersController controller)
LayersController _controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null) if (sourceLayer == null)
{ {
throw new Exception("SourceLayer record not found"); throw new Exception("SourceLayer record not found");
} }
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer); var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
if (sourceImportWorker == null) if (sourceImportWorker == null)
{ {
throw new Exception("SourceImportWorkerL layer not found"); throw new Exception("SourceImportWorkerL layer not found");
} }
string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null) if (sourceLayer == null)
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id) .Where(x => x.ParentId == processWorker.Id)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed, ParentId = processWorker.Id,
ParentId = processWorker!.Id, Number = _db.Layers.Count() + 1
Number = db.Layers.Count() + 1,
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T4"; processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T4";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -68,14 +59,12 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var dataSource = _db.Layers
Layer? dataSource = db.Layers
.Include(x => x.Records) .Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id .Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
@@ -87,9 +76,7 @@ namespace WebAPI.dataProcessors
throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); throw new Exception($"DataSource not found, {sourceImportWorker.Name}");
} }
foreach (Record record in dataSource.Records!) var newRecords = dataSource.Records!.Select(record => new Record
{
Record processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = record.Code, Code = record.Code,
@@ -97,20 +84,18 @@ namespace WebAPI.dataProcessors
Value1 = record.Value1, Value1 = record.Value1,
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; })
newRecords.Add(processedRecord); .ToList();
}
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} else } else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
} }
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges(); _db.SaveChanges();
} }
}
} }

39
WebAPI/dataProcessors/t4.r2.processor.cs
View File

@@ -1,21 +1,19 @@
using DiunaBIWebAPI.dataProcessors; using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.dataProcessors namespace WebAPI.dataProcessors;
public class T4R2Processor
{ {
public class T4R2Processor
{
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly LayersController _controller; private readonly LayersController _controller;
private readonly LogsController _logsController; private readonly LogsController _logsController;
public T4R2Processor( public T4R2Processor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController controller, LayersController controller,
LogsController logsController) LogsController logsController)
{ {
@@ -53,8 +51,7 @@ namespace WebAPI.dataProcessors
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Type = LayerType.Processed,
Type = LayerType.processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1
}; };
@@ -65,7 +62,6 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
} }
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
@@ -86,7 +82,7 @@ namespace WebAPI.dataProcessors
{ {
var monthCopy = month; var monthCopy = month;
var dataSource = _db.Layers.Where(x => var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T")
) )
@@ -94,7 +90,7 @@ namespace WebAPI.dataProcessors
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
List<Record> news = dataSource.Records! var news = dataSource.Records!
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
.Select(x => .Select(x =>
{ {
@@ -117,8 +113,8 @@ namespace WebAPI.dataProcessors
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -127,7 +123,7 @@ namespace WebAPI.dataProcessors
// year summery // year summery
var dataSourceSum = _db.Layers.Where(x => var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
) )
@@ -136,18 +132,10 @@ namespace WebAPI.dataProcessors
if (dataSourceSum != null) if (dataSourceSum != null)
{ {
var news = dataSourceSum.Records! var news = dataSourceSum.Records!
.Where(x => .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Select(x => .Select(x =>
{ {
Record newRecord = new Record var newRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = x.Code + "13", Code = x.Code + "13",
@@ -165,8 +153,8 @@ namespace WebAPI.dataProcessors
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.Warning,
LogType = LogType.process, LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found", Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow CreatedAt = DateTime.UtcNow
}); });
@@ -185,5 +173,4 @@ namespace WebAPI.dataProcessors
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges(); _db.SaveChanges();
} }
}
} }