mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code refactor finished

Browse Source
This commit is contained in:
Michał Zieliski
2024-06-18 22:02:19 +02:00
parent 7e8406dc9b
commit 763b4a8c4d
19 changed files with 1105 additions and 1228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
WebAPI/Controllers/AdminController.cs
View File

@@ -85,8 +85,8 @@ public class AdminController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "Backup success",
Type = LogEntryType.info,
LogType = LogType.backup,
Type = LogEntryType.Info,
LogType = LogType.Backup,
CreatedAt = DateTime.UtcNow
});
return Ok();
@@ -96,8 +96,8 @@ public class AdminController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "Backup error",
Type = LogEntryType.error,
LogType = LogType.backup,
Type = LogEntryType.Error,
LogType = LogType.Backup,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});

120
WebAPI/Controllers/LayersController.cs
View File

@@ -90,8 +90,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong apiKey ({number})",
Type = LogEntryType.warning,
LogType = LogType.powerBI,
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
return Unauthorized();
@@ -105,8 +105,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - no authorization header ({number})",
Type = LogEntryType.warning,
LogType = LogType.powerBI,
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
return Unauthorized();
@@ -118,8 +118,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong auth header format ({number})",
Type = LogEntryType.warning,
LogType = LogType.powerBI,
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
return Unauthorized();
@@ -133,8 +133,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - bad credentials ({number})",
Type = LogEntryType.warning,
LogType = LogType.powerBI,
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
return Unauthorized();
@@ -143,8 +143,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Sending data for layer {number}",
Type = LogEntryType.info,
LogType = LogType.powerBI,
Type = LogEntryType.Info,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
@@ -157,8 +157,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = e.ToString(),
Type = LogEntryType.error,
LogType = LogType.powerBI,
Type = LogEntryType.Error,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
return BadRequest(e.ToString());
@@ -213,8 +213,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "No Layers to import.",
Type = LogEntryType.info,
LogType = LogType.import,
Type = LogEntryType.Info,
LogType = LogType.Import,
CreatedAt = DateTime.UtcNow
});
return Ok();
@@ -235,8 +235,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.import,
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -267,8 +267,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.import,
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -282,8 +282,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.import,
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "Success (reimported)",
CreatedAt = DateTime.UtcNow
});
@@ -293,8 +293,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.import,
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "importLayer records are up of date. Not processed.",
CreatedAt = DateTime.UtcNow
});
@@ -306,8 +306,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.error,
LogType = LogType.import,
Type = LogEntryType.Error,
LogType = LogType.Import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
@@ -321,8 +321,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.import,
Type = LogEntryType.Error,
LogType = LogType.Import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
@@ -363,8 +363,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"Processing: {type}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
CreatedAt = DateTime.UtcNow
});
@@ -383,8 +383,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "No Layers to process.",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
CreatedAt = DateTime.UtcNow
});
}
@@ -400,8 +400,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.error,
LogType = LogType.process,
Type = LogEntryType.Error,
LogType = LogType.Process,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
@@ -413,8 +413,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.error,
LogType = LogType.process,
Type = LogEntryType.Error,
LogType = LogType.Process,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
@@ -445,14 +445,14 @@ public class LayersController : Controller
case "T3-SourceYearSummary":
{
var processor =
new T3SourceYearSummaryProcessor(_db, _googleSheetValues, this);
processor.process(processWorker);
new T3SourceYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -461,14 +461,14 @@ public class LayersController : Controller
case "T3-MultiSourceYearSummary":
{
var processor =
new T3MultiSourceYearSummaryProcessor(_db, _googleSheetValues, this, _logsController);
processor.process(processWorker);
new T3MultiSourceYearSummaryProcessor(_db, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -477,14 +477,14 @@ public class LayersController : Controller
case "T3-MultiSourceCopySelectedCodesYearSummary":
{
var processor =
new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, _googleSheetValues, this);
processor.process(processWorker);
new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -498,8 +498,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -507,14 +507,14 @@ public class LayersController : Controller
}
case "T4-R2":
{
var processor = new T4R2Processor(_db, _googleSheetValues, this, _logsController);
var processor = new T4R2Processor(_db, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
@@ -531,28 +531,28 @@ public class LayersController : Controller
switch (processType!)
{
case "T3-SingleSource":
var t3SingleSource = new T3SingleSourceProcessor(_db, _googleSheetValues, this);
var t3SingleSource = new T3SingleSourceProcessor(_db, this);
{
t3SingleSource.process(processWorker);
t3SingleSource.Process(processWorker);
break;
}
case "T4-SingleSource":
{
var t4SingleSource = new T4SingleSourceProcessor(_db, _googleSheetValues, this);
t4SingleSource.process(processWorker);
var t4SingleSource = new T4SingleSourceProcessor(_db, this);
t4SingleSource.Process(processWorker);
break;
}
case "T3-MultiSourceSummary":
{
var t3MultiSourceSummary =
new T3MultiSourceSummaryProcessor(_db, _googleSheetValues, this, _logsController);
t3MultiSourceSummary.process(processWorker);
new T3MultiSourceSummaryProcessor(_db, this, _logsController);
t3MultiSourceSummary.Process(processWorker);
break;
}
case "T3-MultiSourceCopySelectedCodes":
{
var t3MultiSourceCopySelectedCode =
new T3MultiSourceCopySelectedCodesProcessor(_db, _googleSheetValues, this);
new T3MultiSourceCopySelectedCodesProcessor(_db, this);
t3MultiSourceCopySelectedCode.Process(processWorker);
break;
}
@@ -561,8 +561,8 @@ public class LayersController : Controller
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});

8
WebAPI/Controllers/LogsController.cs
View File

@@ -28,10 +28,10 @@ public class LogsController : Controller
var type = entry.LogType switch
{
LogType.import => "Import",
LogType.backup => "Backup",
LogType.process => "Process",
LogType.powerBI => "PowerBIAccess",
LogType.Import => "Import",
LogType.Backup => "Backup",
LogType.Process => "Process",
LogType.PowerBi => "PowerBIAccess",
_ => "Other"
};
var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();

6
WebAPI/Exports/googleSheet.export.cs
View File

@@ -32,19 +32,19 @@ public class GoogleSheetExport
switch (layer.Type)
{
case LayerType.import:
case LayerType.Import:
{
data.Add(new List<object> { "Code", "Value1" });
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Value1! }));
break;
}
case LayerType.administration:
case LayerType.Administration:
{
data.Add(new List<object> { "Code", "Desc1"});
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Desc1! }));
break;
}
case LayerType.processed:
case LayerType.Processed:
{
data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3",
"Value5", "Value6", "Value7", "Value8", "Value9", "Value10",

42
WebAPI/Models/Layer.cs
View File

@@ -1,44 +1,42 @@
using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models
namespace WebAPI.Models;
public enum LayerType
{
Import,
Processed,
Administration
}
public class Layer
{
public enum LayerType
{
import,
processed,
administration,
}
public class Layer
{
#region Properties
[Key]
public Guid Id { get; set; }
public Guid Id { get; init; }
[Required]
public int Number { get; set; }
[Required]
public string? Source { get; set; }
public int Number { get; init; }
[Required]
[MaxLength(50)]
public string? Name { get; set; }
[Required]
public LayerType Type { get; set; }
public LayerType Type { get; init; }
[Required]
public DateTime CreatedAt { get; set; }
[Required]
public DateTime ModifiedAt { get; set; }
[Required]
public bool IsDeleted { get; set; }
public bool IsDeleted { get; init; }
#endregion
#region Relations
public ICollection<Record>? Records { get; set; }
public ICollection<ProcessSource>? Sources { get; set; }
public ICollection<Record>? Records { get; init; }
[Required]
public Guid CreatedById { get; set; }
public User? CreatedBy { get; set; }
public User? CreatedBy { get; init; }
[Required]
public Guid ModifiedById { get; set; }
public User? ModifiedBy { get; set; }
public Guid? ParentId { get; set; }
public Layer? Parent { get; set; }
public User? ModifiedBy { get; init; }
public Guid? ParentId { get; init; }
public Layer? Parent { get; init; }
#endregion
}
}

41
WebAPI/Models/LogEntry.cs
View File

@@ -1,25 +1,22 @@
using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models;
namespace WebAPI.Models
public enum LogEntryType
{
public enum LogEntryType
{
info,
warning,
error,
}
public enum LogType {
import,
backup,
process,
powerBI
}
public class LogEntry
{
public LogType LogType { get; set; }
public LogEntryType Type { get; set; }
public string? Message { get; set; }
public string? Title {get; set;}
public DateTime CreatedAt { get; set; }
}
Info,
Warning,
Error
}
public enum LogType {
Import,
Backup,
Process,
PowerBi
}
public class LogEntry
{
public LogType LogType { get; init; }
public LogEntryType Type { get; init; }
public string? Message { get; init; }
public string? Title {get; init;}
public DateTime CreatedAt { get; init; }
}

13
WebAPI/Models/ProcessSource.cs
View File

@@ -1,15 +1,14 @@
using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models
namespace WebAPI.Models;
public class ProcessSource
{
public class ProcessSource
{
#region Relations
[Required]
public Guid LayerId { get; set; }
public Guid LayerId { get; init; }
[Required]
public Guid SourceId { get; set; }
public Layer? Source { get; set; }
public Guid SourceId { get; init; }
public Layer? Source { get; init; }
#endregion
}
}

14
WebAPI/Models/Record.cs
View File

@@ -6,10 +6,10 @@ public class Record
{
#region Properties
[Key]
public Guid Id { get; set; }
public Guid Id { get; init; }
[Required]
[StringLength(50)]
public string? Code { get; set; }
public string? Code { get; init; }
public double? Value1 { get; set; }
public double? Value2 { get; set; }
public double? Value3 { get; set; }
@@ -44,15 +44,15 @@ public class Record
public double? Value32 { get; set; }
//Description fields
[StringLength(50)]
public string? Desc1 { get; set; }
public string? Desc1 { get; init; }
[StringLength(50)]
public string? Desc2 { get; set; }
public string? Desc2 { get; init; }
[StringLength(50)]
public string? Desc3 { get; set; }
public string? Desc3 { get; init; }
[StringLength(50)]
public string? Desc4 { get; set; }
public string? Desc4 { get; init; }
[StringLength(50)]
public string? Desc5 { get; set; }
public string? Desc5 { get; init; }
[StringLength(50)]
public DateTime CreatedAt { get; set; }
public DateTime ModifiedAt { get; set; }

3
WebAPI/dataImporters/morska.fk2.importer.cs
View File

@@ -80,10 +80,9 @@ public class MorskaFk2Importer
}
var layer = new Layer
{
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,

3
WebAPI/dataImporters/morska.importer.cs
View File

@@ -80,10 +80,9 @@ public class MorskaImporter
}
var layer = new Layer
{
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,

17
WebAPI/dataProcessors/t1.r1.processor.cs
View File

@@ -55,8 +55,7 @@ public class T1R1Processor
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
@@ -66,7 +65,7 @@ public class T1R1Processor
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
@@ -78,7 +77,7 @@ public class T1R1Processor
{
var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
x.Type == LayerType.Processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T3")
)
@@ -93,8 +92,8 @@ public class T1R1Processor
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
@@ -102,7 +101,7 @@ public class T1R1Processor
}
// year summary
var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
x.Type == LayerType.Processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
@@ -117,8 +116,8 @@ public class T1R1Processor
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});

53
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs
View File

@@ -1,32 +1,27 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3MultiSourceCopySelectedCodesProcessor
{
public class T3MultiSourceCopySelectedCodesProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly AppDbContext _db;
private readonly LayersController _controller;
public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
AppDbContext db,
LayersController controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
_db = db;
_controller = controller;
}
public void Process(Layer processWorker)
{
var year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (!sources!.Any())
{
@@ -40,8 +35,8 @@ namespace WebAPI.dataProcessors
var codesList = ProcessHelper.ParseCodes(codes);
var processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
@@ -53,10 +48,9 @@ namespace WebAPI.dataProcessors
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -64,18 +58,18 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
var dataSources = sources!.Select(source => db.Layers.Where(x => x.Type == LayerType.processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3"))
var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3"))
.Include(x => x.Records)
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
throw new Exception("DataSources are empty");
}
@@ -100,14 +94,13 @@ namespace WebAPI.dataProcessors
.ToList();
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_db.Layers.Update(processedLayer);
}
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

73
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs
View File

@@ -1,49 +1,43 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor
{
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly AppDbContext _db;
private readonly LayersController _controller;
public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
AppDbContext db,
LayersController controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
_db = db;
_controller = controller;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -52,25 +46,25 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>();
var dataSources = new List<Layer>();
for (int i = 1; i < 13; i++)
for (var i = 1; i < 13; i++)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed
var i1 = i;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed
&& !x.IsDeleted
&& x.Name != null && x.Name.Contains($"{year}/{i}-AB-T3"))
&& x.Name != null && x.Name.Contains($"{year}/{i1}-AB-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource!);
dataSources.Add(dataSource);
}
}
@@ -79,12 +73,12 @@ namespace WebAPI.dataProcessors
throw new Exception("DataSources are empty");
}
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!)
foreach (var baseRecord in dataSources.Last().Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
var processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = baseRecord.Code,
@@ -102,15 +96,14 @@ namespace WebAPI.dataProcessors
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
_db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

135
WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
View File

@@ -1,58 +1,53 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Calculator;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3MultiSourceSummaryProcessor
{
public class T3MultiSourceSummaryProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
private readonly AppDbContext _db;
private readonly LayersController _controller;
private readonly LogsController _logsController;
public T3MultiSourceSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller,
LogsController _logsController)
AppDbContext db,
LayersController controller,
LogsController logsController)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
logsController = _logsController;
_db = db;
_controller = controller;
_logsController = logsController;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (!sources!.Any())
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -60,44 +55,33 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>();
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name!=null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3")
)
var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource);
}
}
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
throw new Exception("DataSources are empty");
}
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
List<string> baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (string baseCode in baseCodess)
foreach (var baseCode in baseCodes)
{
List<Record> codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode)
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
Record processedRecord = new Record
var processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = $"9{baseCode}",
@@ -113,33 +97,33 @@ namespace WebAPI.dataProcessors
}
// Dynamic Codes
List<Record>? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
if (dynamicCodes != null && dynamicCodes.Any())
{
foreach (Record dynamicCode in dynamicCodes)
foreach (var dynamicCode in dynamicCodes)
{
try
{
if (dynamicCode.Desc1 == null)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
continue;
}
BaseCalc calc = new BaseCalc(dynamicCode.Desc1);
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
@@ -152,11 +136,11 @@ namespace WebAPI.dataProcessors
}
catch (Exception e)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
@@ -164,11 +148,11 @@ namespace WebAPI.dataProcessors
}
catch (Exception e)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
@@ -179,14 +163,13 @@ namespace WebAPI.dataProcessors
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_db.Layers.Update(processedLayer);
}
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

161
WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs
View File

@@ -1,57 +1,52 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Calculator;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3MultiSourceYearSummaryProcessor
{
public class T3MultiSourceYearSummaryProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
private readonly AppDbContext _db;
private readonly LayersController _controller;
private readonly LogsController _logsController;
public T3MultiSourceYearSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller,
LogsController _logsController)
AppDbContext db,
LayersController controller,
LogsController logsController)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
logsController = _logsController;
_db = db;
_controller = controller;
_logsController = logsController;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (!sources!.Any())
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -59,73 +54,48 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>();
foreach (Record source in sources!)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
var dataSources = sources!.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource);
}
}
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
throw new Exception("DataSources are empty");
}
List<Layer> dataSourcesValidation = new List<Layer>();
for (int i = 1; i < 13; i++)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed
&& !x.IsDeleted
&& x.Name!=null && x.Name.Contains($"{year}/{i}-AA-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSourcesValidation.Add(dataSource!);
}
}
if (dataSources.Count == 0)
{
throw new Exception($"DataSourcesValidation are empty");
throw new Exception("DataSourcesValidation are empty");
}
List<Record> allRecords = dataSources
var allRecords = dataSources
.SelectMany(x => x.Records!).ToList();
List<string> baseCodess = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (string baseCode in baseCodess)
foreach (var baseCode in baseCodes)
{
List<Record> codeRecords = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode)
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
List<Record> codeRecordsValidation = allRecords.Where(x =>
x.Code!.Substring(1) == baseCode)
var codeRecordsValidation = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
Record processedRecord = new Record
var processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = $"9{baseCode}",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
Record validationRecord = new Record();
var validationRecord = new Record();
for (var i = 1; i < 33; i++)
{
ProcessHelper.SetValue(processedRecord, i,
@@ -135,10 +105,10 @@ namespace WebAPI.dataProcessors
codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i)));
if (
ProcessHelper.GetValue(processedRecord,i) !=
ProcessHelper.GetValue(validationRecord, i))
double.Abs((double)(ProcessHelper.GetValue(processedRecord,i) -
ProcessHelper.GetValue(validationRecord, i))!) > 0.01)
{
throw new Exception($"ValidationError: Code {baseCode!}, " +
throw new Exception($"ValidationError: Code {baseCode}, " +
$"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " +
$"{ProcessHelper.GetValue(validationRecord, i)})");
}
@@ -149,33 +119,33 @@ namespace WebAPI.dataProcessors
// Dynamic Codes
List<Record>? dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
var dynamicCodes = processWorker.Records?.Where(x => x.Code == "DynamicCode").ToList();
if (dynamicCodes != null && dynamicCodes.Any())
{
foreach (Record dynamicCode in dynamicCodes)
foreach (var dynamicCode in dynamicCodes)
{
try
{
if (dynamicCode.Desc1 == null)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
continue;
}
BaseCalc calc = new BaseCalc(dynamicCode.Desc1);
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
@@ -188,11 +158,11 @@ namespace WebAPI.dataProcessors
}
catch (Exception e)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
@@ -200,11 +170,11 @@ namespace WebAPI.dataProcessors
}
catch (Exception e)
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
@@ -213,14 +183,13 @@ namespace WebAPI.dataProcessors
}
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_db.Layers.Update(processedLayer);
}
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

109
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -1,66 +1,58 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3SingleSourceProcessor
{
public class T3SingleSourceProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly AppDbContext _db;
private readonly LayersController _controller;
public T3SingleSourceProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
AppDbContext db,
LayersController controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
_db = db;
_controller = controller;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null)
{
throw new Exception("SourceLayer record not found");
}
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
}
string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null)
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id)
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -68,14 +60,14 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
List<Layer> dataSources = db.Layers
var dataSources = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted)
@@ -87,13 +79,13 @@ namespace WebAPI.dataProcessors
}
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!)
foreach (var baseRecord in dataSources.Last().Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record
var processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = baseRecord.Code,
@@ -101,51 +93,41 @@ namespace WebAPI.dataProcessors
ModifiedAt = DateTime.UtcNow
};
int lastDayInMonth = DateTime.DaysInMonth(year, month);
double previousValue = 0;
var lastDayInMonth = DateTime.DaysInMonth(year, month);
//day 1
double firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
var firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
ProcessHelper.SetValue(processedRecord, 1, firstVal);
previousValue = firstVal;
var previousValue = firstVal;
//days 2-29/30
for (int i=2; i<lastDayInMonth; i++)
for (var i=2; i<lastDayInMonth; i++)
{
double? dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
var dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1;
if (dayVal == null)
{
//TODO: missing day value? Should I log it?
ProcessHelper.SetValue(processedRecord, i, 0);
} else
{
double processedVal = (dayVal ?? 0) - previousValue;
var processedVal = dayVal - previousValue;
ProcessHelper.SetValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0;
previousValue = (double)dayVal;
}
}
//last day
double? lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
var lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1;
if (lastVal == null)
{
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
} else
{
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
}
// copy last value
double? valueToCopy = codeRecords
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord);
@@ -153,14 +135,13 @@ namespace WebAPI.dataProcessors
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
_db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}
}

75
WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
View File

@@ -1,53 +1,48 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T3SourceYearSummaryProcessor
{
public class T3SourceYearSummaryProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly AppDbContext _db;
private readonly LayersController _controller;
public T3SourceYearSummaryProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller
AppDbContext db,
LayersController controller
)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
_db = db;
_controller = controller;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
string? year = processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
string? source = processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -55,24 +50,25 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
List<Layer> dataSources = new List<Layer>();
for (int i=1; i<13; i++)
var dataSources = new List<Layer>();
for (var i=1; i<13; i++)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed
var i1 = i;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed
&& !x.IsDeleted
&& x.Name!=null && x.Name.Contains($"{year}/{i}-{source}-T3"))
&& x.Name!=null && x.Name.Contains($"{year}/{i1}-{source}-T3"))
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
dataSources.Add(dataSource!);
dataSources.Add(dataSource);
}
}
@@ -81,12 +77,12 @@ namespace WebAPI.dataProcessors
throw new Exception("DataSources are empty");
}
List<Record> allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
foreach (Record baseRecord in dataSources.Last()?.Records!)
foreach (var baseRecord in dataSources.Last().Records!)
{
List<Record> codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
Record processedRecord = new Record
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
var processedRecord = new Record
{
Id = Guid.NewGuid(),
Code = baseRecord.Code,
@@ -103,13 +99,12 @@ namespace WebAPI.dataProcessors
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_db.Layers.Update(processedLayer);
}
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

77
WebAPI/dataProcessors/t4.SingleSource.processor.cs
View File

@@ -1,66 +1,57 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System;
using System.Globalization;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T4SingleSourceProcessor
{
public class T4SingleSourceProcessor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly AppDbContext _db;
private readonly LayersController _controller;
public T4SingleSourceProcessor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller)
AppDbContext db,
LayersController controller)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
_db = db;
_controller = controller;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
int month = int.Parse(processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
string? sourceLayer = processWorker?.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null)
{
throw new Exception("SourceLayer record not found");
}
Layer? sourceImportWorker = db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
}
string? source= processWorker?.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
var source= processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null)
{
throw new Exception("Source record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id)
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month}-{source}-T4";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -68,14 +59,12 @@ namespace WebAPI.dataProcessors
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
Layer? dataSource = db.Layers
var dataSource = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted)
@@ -87,9 +76,7 @@ namespace WebAPI.dataProcessors
throw new Exception($"DataSource not found, {sourceImportWorker.Name}");
}
foreach (Record record in dataSource.Records!)
{
Record processedRecord = new Record
var newRecords = dataSource.Records!.Select(record => new Record
{
Id = Guid.NewGuid(),
Code = record.Code,
@@ -97,20 +84,18 @@ namespace WebAPI.dataProcessors
Value1 = record.Value1,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(processedRecord);
}
})
.ToList();
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
} else
{
db.Layers.Update(processedLayer);
_db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}
}

39
WebAPI/dataProcessors/t4.r2.processor.cs
View File

@@ -1,21 +1,19 @@
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
namespace WebAPI.dataProcessors;
public class T4R2Processor
{
public class T4R2Processor
{
private readonly AppDbContext _db;
private readonly LayersController _controller;
private readonly LogsController _logsController;
public T4R2Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController controller,
LogsController logsController)
{
@@ -53,8 +51,7 @@ namespace WebAPI.dataProcessors
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
@@ -65,7 +62,6 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
@@ -86,7 +82,7 @@ namespace WebAPI.dataProcessors
{
var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
x.Type == LayerType.Processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T")
)
@@ -94,7 +90,7 @@ namespace WebAPI.dataProcessors
.FirstOrDefault();
if (dataSource != null)
{
List<Record> news = dataSource.Records!
var news = dataSource.Records!
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
.Select(x =>
{
@@ -117,8 +113,8 @@ namespace WebAPI.dataProcessors
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
@@ -127,7 +123,7 @@ namespace WebAPI.dataProcessors
// year summery
var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
x.Type == LayerType.Processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
)
@@ -136,18 +132,10 @@ namespace WebAPI.dataProcessors
if (dataSourceSum != null)
{
var news = dataSourceSum.Records!
.Where(x =>
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Record newRecord = new Record
var newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + "13",
@@ -165,8 +153,8 @@ namespace WebAPI.dataProcessors
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
@@ -185,5 +173,4 @@ namespace WebAPI.dataProcessors
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}
}