mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

R2 Processor

Browse Source
This commit is contained in:
Michał Zieliski
2024-06-17 22:28:40 +02:00
parent e0b6f798a1
commit f93bb9cd42
3 changed files with 280 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
WebAPI/Controllers/LayersController.cs
View File

@@ -378,7 +378,8 @@ namespace WebAPI.Controllers
"T3-MultiSourceSummary", // AA
"T3-MultiSourceYearSummary", // AA/13
"T4-SingleSource",
"T3-R1"
"T1-R1",
"T4-R2"
};
foreach (string type in processTypes)
@@ -519,9 +520,9 @@ namespace WebAPI.Controllers
return;
}
if (processType == "T3-R1")
if (processType == "T1-R1")
{
T3R1Processor processor = new T3R1Processor(db, googleSheetValues, this, logsController);
T1R1Processor processor = new T1R1Processor(db, googleSheetValues, this, logsController);
processor.process(processWorker!);
logsController.AddEntry(new LogEntry
@@ -534,6 +535,22 @@ namespace WebAPI.Controllers
});
return;
}
if (processType == "T4-R2")
{
T4R2Processor processor = new T4R2Processor(db, googleSheetValues, this, logsController);
processor.process(processWorker!);
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.info,
LogType = LogType.process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
string? month = processWorker?.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1;
if (month == null)

4
WebAPI/dataProcessors/t1.r1.processor.cs
View File

@@ -8,14 +8,14 @@ using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class T3R1Processor
public class T1R1Processor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
public T3R1Processor(
public T1R1Processor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller,

258
WebAPI/dataProcessors/t4.r2.processor.cs Normal file
View File

@@ -0,0 +1,258 @@
using System.Globalization;
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers;
using WebAPI.Models;
namespace WebAPI.dataProcessors
{
public class T4R2Processor
{
private readonly AppDbContext db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
private readonly LogsController logsController;
public T4R2Processor(
AppDbContext _db,
SpreadsheetsResource.ValuesResource _googleSheetValues,
LayersController _controller,
LogsController _logsController)
{
db = _db;
googleSheetValues = _googleSheetValues;
controller = _controller;
logsController = _logsController;
}
public void process(Layer processWorker)
{
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0)
{
throw new Exception("Source record not found");
}
string? layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1;
if (layerName == null)
{
throw new Exception("LayerName record not found");
}
Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id
&& !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
bool isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Source = "",
Type = LayerType.processed,
ParentId = processWorker!.Id,
Number = db.Layers.Count() + 1,
};
processedLayer.Name = $"L{processedLayer.Number}-{layerName}";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
}
processedLayer.Sources = new List<ProcessSource>();
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>();
foreach (Record source in sources!)
{
string? rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}")
?.Desc1;
List<int> sourceCodes = new List<int>();
if (rawSourceCodes != null)
{
sourceCodes = ProcessHelper.parseCodes(rawSourceCodes);
}
for (int month = 1; month <= DateTime.UtcNow.Month; month++)
{
Layer? dataSource = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSource != null)
{
List<Record> news = dataSource.Records!
.Where(x =>
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Select(x =>
{
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + month.ToString("D2"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1,
Desc1 = x.Desc1
};
return newRecord;
}
).ToList();
newRecords.AddRange(news);
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
}
}
// year summery
Layer? dataSourceSum = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSourceSum != null)
{
List<Record> news = dataSourceSum.Records!
.Where(x =>
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Select(x =>
{
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + "13",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
};
return newRecord;
}
).ToList();
newRecords.AddRange(news);
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
}
}
/*
// year summary
Layer? dataSourceSum = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSourceSum != null)
{
dataSources.Add(dataSourceSum);
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
}
}
/*
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
}
List<Record> newRecords = dataSources
.SelectMany(x => x.Records!)
.Where(x => codesList.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Layer? layer = dataSources.SingleOrDefault(y => y.Id == x.LayerId);
string postFix = layer!.Name!.Split("/")[1].Split("-")[0];
if (postFix.Length == 1)
{
postFix = "0" + postFix;
}
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + postFix,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
};
return newRecord;
})
.ToList();
*/
if (isNew)
{
db.Layers.Add(processedLayer);
}
else
{
db.Layers.Update(processedLayer);
}
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
}
}
}