mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: refactor

Browse Source
This commit is contained in:
Michał Zieliski
2024-06-18 19:40:16 +02:00
parent c2a98e0386
commit a0a228f86d
24 changed files with 1415 additions and 1654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
WebAPI/dataProcessors/ProcessHelper.cs
View File

@@ -1,209 +1,175 @@
using System;
using WebAPI.Models;
using WebAPI.Models;
namespace DiunaBIWebAPI.dataProcessors
namespace DiunaBIWebAPI.dataProcessors;
public static class ProcessHelper
{
public static class ProcessHelper
{
public static void setValue(Record record, int number, double? value)
public static void SetValue(Record record, int number, double? value)
{
value = (double)Math.Round((decimal)(value ?? 0), 2);
switch (number)
{
value = (double)Math.Round((decimal)(value ?? 0), 2);
switch (number)
case 1:
record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
public static double? getValue(Record record, int number)
{
return number switch
{
1 => record.Value1,
2 => record.Value2,
3 => record.Value3,
4 => record.Value4,
5 => record.Value5,
6 => record.Value6,
7 => record.Value7,
8 => record.Value8,
9 => record.Value9,
10 => record.Value10,
11 => record.Value11,
12 => record.Value12,
13 => record.Value13,
14 => record.Value14,
15 => record.Value15,
16 => record.Value16,
17 => record.Value17,
18 => record.Value18,
19 => record.Value19,
20 => record.Value20,
21 => record.Value21,
22 => record.Value22,
23 => record.Value23,
24 => record.Value24,
25 => record.Value25,
26 => record.Value26,
27 => record.Value27,
28 => record.Value28,
29 => record.Value29,
30 => record.Value30,
31 => record.Value31,
32 => record.Value32,
_ => null
};
}
public static List<int> ParseCodes(string codes)
{
var codesList = new List<int>();
foreach (var code in codes.Split(';'))
{
var range = code.Split('-');
switch (range.Length)
{
case 1:
record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
public static double? getValue(Record record, int number)
{
switch (number)
{
case 1:
return record.Value1;
case 2:
return record.Value2;
case 3:
return record.Value3;
case 4:
return record.Value4;
case 5:
return record.Value5;
case 6:
return record.Value6;
case 7:
return record.Value7;
case 8:
return record.Value8;
case 9:
return record.Value9;
case 10:
return record.Value10;
case 11:
return record.Value11;
case 12:
return record.Value12;
case 13:
return record.Value13;
case 14:
return record.Value14;
case 15:
return record.Value15;
case 16:
return record.Value16;
case 17:
return record.Value17;
case 18:
return record.Value18;
case 19:
return record.Value19;
case 20:
return record.Value20;
case 21:
return record.Value21;
case 22:
return record.Value22;
case 23:
return record.Value23;
case 24:
return record.Value24;
case 25:
return record.Value25;
case 26:
return record.Value26;
case 27:
return record.Value27;
case 28:
return record.Value28;
case 29:
return record.Value29;
case 30:
return record.Value30;
case 31:
return record.Value31;
case 32:
return record.Value32;
default:
return null;
}
}
public static List<int> parseCodes(string codes)
{
List<int> codesList = new List<int>();
foreach (string code in codes.Split(';'))
{
string[] range = code.Split('-');
if (range.Length == 1)
{
codesList.Add(int.Parse(range[0]));
}
else if (range.Length == 2)
break;
case 2:
{
for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++)
{
codesList.Add(i);
}
}
else
{
throw new Exception($"Invalid code range: {code}");
}
}
return codesList;
}
}
}
break;
}
default:
throw new Exception($"Invalid code range: {code}");
}
}
return codesList;
}
}

2
WebAPI/dataProcessors/t1.r1.processor.cs
View File

@@ -40,7 +40,7 @@ namespace WebAPI.dataProcessors
{
throw new Exception("Codes record not found");
}
List<int> codesList = ProcessHelper.parseCodes(codes);
List<int> codesList = ProcessHelper.ParseCodes(codes);
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id

4
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodes.processor.cs
View File

@@ -38,7 +38,7 @@ namespace WebAPI.dataProcessors
throw new Exception("Codes record not found");
}
// create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512]
List<int> codesList = ProcessHelper.parseCodes(codes);
List<int> codesList = ProcessHelper.ParseCodes(codes);
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
@@ -103,7 +103,7 @@ namespace WebAPI.dataProcessors
};
for (var i = 1; i < 33; i++)
{
ProcessHelper.setValue(newRecord, i, ProcessHelper.getValue(x, i));
ProcessHelper.SetValue(newRecord, i, ProcessHelper.getValue(x, i));
}
return newRecord;
})

2
WebAPI/dataProcessors/t3.MultiSourceCopySelectedCodesYearSummary.processor.cs
View File

@@ -93,7 +93,7 @@ namespace WebAPI.dataProcessors
};
for (var i = 1; i < 33; i++)
{
ProcessHelper.setValue(processedRecord, i,
ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
}

2
WebAPI/dataProcessors/t3.MultiSourceSummary.processor.cs
View File

@@ -106,7 +106,7 @@ namespace WebAPI.dataProcessors
};
for (var i = 1; i<33; i++)
{
ProcessHelper.setValue(processedRecord, i,
ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
}
newRecords.Add(processedRecord);

4
WebAPI/dataProcessors/t3.MultiSourceYearSummary.processor.cs
View File

@@ -128,10 +128,10 @@ namespace WebAPI.dataProcessors
Record validationRecord = new Record();
for (var i = 1; i < 33; i++)
{
ProcessHelper.setValue(processedRecord, i,
ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
ProcessHelper.setValue(validationRecord, i,
ProcessHelper.SetValue(validationRecord, i,
codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i)));
if (

12
WebAPI/dataProcessors/t3.SingleSource.processor.cs
View File

@@ -108,7 +108,7 @@ namespace WebAPI.dataProcessors
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0;
ProcessHelper.setValue(processedRecord, 1, firstVal);
ProcessHelper.SetValue(processedRecord, 1, firstVal);
previousValue = firstVal;
//days 2-29/30
for (int i=2; i<lastDayInMonth; i++)
@@ -120,11 +120,11 @@ namespace WebAPI.dataProcessors
if (dayVal == null)
{
//TODO: missing day value? Should I log it?
ProcessHelper.setValue(processedRecord, i, 0);
ProcessHelper.SetValue(processedRecord, i, 0);
} else
{
double processedVal = (dayVal ?? 0) - previousValue;
ProcessHelper.setValue(processedRecord, i, processedVal);
ProcessHelper.SetValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0;
}
}
@@ -136,17 +136,17 @@ namespace WebAPI.dataProcessors
if (lastVal == null)
{
ProcessHelper.setValue(processedRecord, lastDayInMonth, 0);
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
} else
{
ProcessHelper.setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
}
// copy last value
double? valueToCopy = codeRecords
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1;
ProcessHelper.setValue(processedRecord, 32, valueToCopy);
ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord);
}

2
WebAPI/dataProcessors/t3.SourceYearSummary.processor.cs
View File

@@ -95,7 +95,7 @@ namespace WebAPI.dataProcessors
};
for (var i = 1; i < 33; i++)
{
ProcessHelper.setValue(processedRecord, i,
ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
}
newRecords.Add(processedRecord);

155
WebAPI/dataProcessors/t4.r2.processor.cs
View File

@@ -1,8 +1,5 @@
using System.Globalization;
using DiunaBIWebAPI.dataProcessors;
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
@@ -12,46 +9,44 @@ namespace WebAPI.dataProcessors
{
public class T4R2Processor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
private readonly AppDbContext _db;
private readonly LayersController _controller;
private readonly LogsController _logsController;
public T4R2Processor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller,
LogsController _logsController)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController controller,
LogsController logsController)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
logsController = _logsController;
_db = db;
_controller = controller;
_logsController = logsController;
}
public void process(Layer processWorker)
public void Process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (!sources!.Any())
{
throw new Exception("Source record not found");
}
string? layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1;
var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1;
if (layerName == null)
{
throw new Exception("LayerName record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
var isNew = false;
if (processedLayer == null)
{
isNew = true;
@@ -60,8 +55,8 @@ namespace WebAPI.dataProcessors
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
};
processedLayer.Name = $"L{processedLayer.Number}-{layerName}";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -75,42 +70,35 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
var newRecords = new List<Record>();
foreach (Record source in sources!)
foreach (var source in sources!)
{
string? rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}")
var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}")
?.Desc1;
List<int> sourceCodes = new List<int>();
var sourceCodes = new List<int>();
if (rawSourceCodes != null)
{
sourceCodes = ProcessHelper.parseCodes(rawSourceCodes);
sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes);
}
for (int month = 1; month <= DateTime.UtcNow.Month; month++)
for (var month = 1; month <= DateTime.UtcNow.Month; month++)
{
Layer? dataSource = db.Layers.Where(x =>
var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T")
x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
List<Record> news = dataSource.Records!
.Where(x =>
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Record newRecord = new Record
var newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + month.ToString("D2"),
@@ -126,9 +114,9 @@ namespace WebAPI.dataProcessors
}
else
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
@@ -138,7 +126,7 @@ namespace WebAPI.dataProcessors
}
// year summery
Layer? dataSourceSum = db.Layers.Where(x =>
var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
@@ -147,7 +135,7 @@ namespace WebAPI.dataProcessors
.FirstOrDefault();
if (dataSourceSum != null)
{
List<Record> news = dataSourceSum.Records!
var news = dataSourceSum.Records!
.Where(x =>
{
if (sourceCodes.Count > 0)
@@ -174,9 +162,9 @@ namespace WebAPI.dataProcessors
}
else
{
logsController.AddEntry(new LogEntry
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
@@ -184,75 +172,18 @@ namespace WebAPI.dataProcessors
});
}
}
/*
// year summary
Layer? dataSourceSum = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSourceSum != null)
{
dataSources.Add(dataSourceSum);
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
}
}
/*
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
}
List<Record> newRecords = dataSources
.SelectMany(x => x.Records!)
.Where(x => codesList.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Layer? layer = dataSources.SingleOrDefault(y => y.Id == x.LayerId);
string postFix = layer!.Name!.Split("/")[1].Split("-")[0];
if (postFix.Length == 1)
{
postFix = "0" + postFix;
}
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + postFix,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
};
return newRecord;
})
.ToList();
*/
if (isNew)
{
db.Layers.Add(processedLayer);
_db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
_db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
_controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}
}