WIP: refactor

This commit is contained in:
Michał Zieliski
2024-06-18 19:40:16 +02:00
parent c2a98e0386
commit a0a228f86d
24 changed files with 1415 additions and 1654 deletions

View File

@@ -1,37 +1,35 @@
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using System.Collections.Generic;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI namespace WebAPI;
public class AppDbContext : DbContext
{ {
public class AppDbContext : DbContext public DbSet<User> Users { get; init; }
public DbSet<Layer> Layers { get; init; }
public DbSet<Record> Records { get; init; }
public DbSet<ProcessSource> ProcessSources { get; init; }
public AppDbContext(DbContextOptions<AppDbContext> options) : base(options)
{ {
public DbSet<User> Users { get; init; } }
public DbSet<Layer> Layers { get; init; }
public DbSet<Record> Records { get; init; }
public DbSet<ProcessSource> ProcessSources { get; init; }
public AppDbContext(DbContextOptions<AppDbContext> options) : base(options) protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<ProcessSource>().HasKey(x => new
{ {
} x.LayerId,
x.SourceId
});
}
protected override void OnModelCreating(ModelBuilder modelBuilder) private static readonly LoggerFactory MyLoggerFactory =
{ new(new[] {
modelBuilder.Entity<ProcessSource>().HasKey(x => new new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider()
{
x.LayerId,
x.SourceId
});
}
private static readonly LoggerFactory MyLoggerFactory =
new(new[] {
new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider()
}); });
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{ {
optionsBuilder.UseLoggerFactory(MyLoggerFactory); optionsBuilder.UseLoggerFactory(MyLoggerFactory);
}
} }
} }

View File

@@ -69,7 +69,7 @@ public class BaseCalc
} }
Entity expr = formula; Entity expr = formula;
ProcessHelper.setValue(result, i, (double)expr.EvalNumerical()); ProcessHelper.SetValue(result, i, (double)expr.EvalNumerical());
} }
return result; return result;
} }

View File

@@ -3,105 +3,105 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Data.SqlClient; using Microsoft.Data.SqlClient;
using WebAPI.Models; using WebAPI.Models;
using static Google.Apis.Drive.v3.FilesResource;
namespace WebAPI.Controllers namespace WebAPI.Controllers;
[ApiController]
[Route("api/[controller]")]
public class AdminController : Controller
{ {
[ApiController] private readonly GoogleDriveHelper _googleDriveHelper;
[Route("api/[controller]")] private readonly IConfiguration _configuration;
public class AdminController : Controller private readonly LogsController _logsController;
{
private readonly GoogleDriveHelper _googleDriveHelper;
private readonly IConfiguration _configuration;
private readonly LogsController _logsController;
public AdminController( public AdminController(
GoogleDriveHelper googleDriveHelper, GoogleDriveHelper googleDriveHelper,
IConfiguration configuration, IConfiguration configuration,
LogsController logsController) LogsController logsController)
{
_googleDriveHelper = googleDriveHelper;
_configuration = configuration;
_logsController = logsController;
}
[HttpGet]
[Route("BackupDatabase/{apiKey}")]
[AllowAnonymous]
public IActionResult BackupDatabase(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{ {
_googleDriveHelper = googleDriveHelper; return Unauthorized();
_configuration = configuration;
_logsController = logsController;
} }
[HttpGet] try
[Route("BackupDatabase/{apiKey}")]
[AllowAnonymous]
public IActionResult BackupDatabase(string apiKey)
{ {
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) const string databaseName = "diunabi-morska";
var localDatabasePath = $"{_configuration["dbBackupFile"]}-{DateTime.UtcNow.Day}.bak";
const string formatMediaName = $"DatabaseToolkitBackup_{databaseName}";
const string formatName = $"Full Backup of {databaseName}";
var connection = new SqlConnection(_configuration.GetConnectionString("SQLDatabase"));
const string sql = """
BACKUP DATABASE @databaseName
TO DISK = @localDatabasePath
WITH FORMAT,
MEDIANAME = @formatMediaName,
NAME = @formatName
""";
connection.Open();
var command = new SqlCommand(sql, connection);
command.CommandType = CommandType.Text;
command.CommandTimeout = 7200;
command.Parameters.AddWithValue("@databaseName", databaseName);
command.Parameters.AddWithValue("@localDatabasePath", localDatabasePath);
command.Parameters.AddWithValue("@formatMediaName", formatMediaName);
command.Parameters.AddWithValue("@formatName", formatName);
command.ExecuteNonQuery();
var body = new Google.Apis.Drive.v3.Data.File
{ {
return Unauthorized(); Name = Path.GetFileName(localDatabasePath),
Parents = new List<string?> { _configuration["GDriveBackupDirectory"] },
MimeType = "application/octet-stream"
};
var fsSource = new FileStream(localDatabasePath, FileMode.Open, FileAccess.Read);
if (_googleDriveHelper.Service is null)
{
throw new Exception("Google Drive API not initialized");
} }
try var request = _googleDriveHelper.Service.Files.Create(body, fsSource, body.MimeType);
{ request.Fields = "id";
var databaseName = "diunabi-morska";
var localDatabasePath = $"{_configuration["dbBackupFile"]}-{DateTime.UtcNow.Day}.bak";
var formatMediaName = $"DatabaseToolkitBackup_{databaseName}";
var formatName = $"Full Backup of {databaseName}";
var connection = new SqlConnection(_configuration.GetConnectionString("SQLDatabase")); request.Upload();
var sql = @"BACKUP DATABASE @databaseName
TO DISK = @localDatabasePath
WITH FORMAT,
MEDIANAME = @formatMediaName,
NAME = @formatName";
connection.Open();
var command = new SqlCommand(sql, connection);
command.CommandType = CommandType.Text;
command.CommandTimeout = 7200;
command.Parameters.AddWithValue("@databaseName", databaseName);
command.Parameters.AddWithValue("@localDatabasePath", localDatabasePath);
command.Parameters.AddWithValue("@formatMediaName", formatMediaName);
command.Parameters.AddWithValue("@formatName", formatName);
command.ExecuteNonQuery();
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File
{
Name = Path.GetFileName(localDatabasePath),
Parents = new List<string?> { _configuration["GDriveBackupDirectory"] },
MimeType = "application/octet-stream"
};
var fsSource = new FileStream(localDatabasePath, FileMode.Open, FileAccess.Read);
if (_googleDriveHelper.Service is null)
{
throw new Exception("Google Drive API not initialized");
}
var request = _googleDriveHelper.Service.Files.Create(body, fsSource, body.MimeType);
request.Fields = "id";
request.Upload();
_logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{
Title = "Backup success",
Type = LogEntryType.info,
LogType = LogType.backup,
CreatedAt = DateTime.UtcNow
});
return Ok();
}
catch (Exception e)
{ {
_logsController.AddEntry(new LogEntry Title = "Backup success",
{ Type = LogEntryType.info,
Title = "Backup error", LogType = LogType.backup,
Type = LogEntryType.error, CreatedAt = DateTime.UtcNow
LogType = LogType.backup, });
Message = e.ToString(), return Ok();
CreatedAt = DateTime.UtcNow }
}); catch (Exception e)
return BadRequest(e.ToString()); {
} _logsController.AddEntry(new LogEntry
{
Title = "Backup error",
Type = LogEntryType.error,
LogType = LogType.backup,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
return BadRequest(e.ToString());
} }
} }
} }

View File

@@ -1,72 +1,58 @@
using Google.Apis.Auth; using Google.Apis.Auth;
using Google.Apis.Http;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Identity.Client.Platforms.Features.DesktopOs.Kerberos;
using Microsoft.IdentityModel.Tokens; using Microsoft.IdentityModel.Tokens;
using System.Configuration;
using System.IdentityModel.Tokens.Jwt; using System.IdentityModel.Tokens.Jwt;
using System.Security.Claims; using System.Security.Claims;
using System.Text; using System.Text;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.Controllers namespace WebAPI.Controllers;
[ApiController]
[Route("api/[controller]")]
// [Authorize]
public class AuthController : Controller
{ {
[ApiController] private readonly AppDbContext _db;
[Route("api/[controller]")] private readonly IConfiguration _configuration;
// [Authorize] public AuthController(
public class AuthController : Controller AppDbContext db, IConfiguration configuration)
{ _db = db; _configuration = configuration; }
[HttpPost]
[Route("apiToken")]
public async Task<IActionResult> ApiToken([FromBody] string credential)
{ {
private readonly AppDbContext db; var settings = new GoogleJsonWebSignature.ValidationSettings
private readonly IConfiguration configuration;
public AuthController(
AppDbContext _db, IConfiguration _configuration)
{ db = _db; configuration = _configuration; }
[HttpPost]
[Route("apiToken")]
public async Task<IActionResult> apiToken([FromBody] string credential)
{ {
var settings = new GoogleJsonWebSignature.ValidationSettings() Audience = new List<string> { _configuration.GetValue<string>("GoogleClientId")! }
{ };
Audience = new List<string> { configuration.GetValue<string>("GoogleClientId")! } var payload = await GoogleJsonWebSignature.ValidateAsync(credential, settings);
}; var user = _db.Users.FirstOrDefault(x => x.Email == payload.Email);
var payload = await GoogleJsonWebSignature.ValidateAsync(credential, settings); return user != null ? (IActionResult)Ok(JwtGenerator(user)) : Unauthorized();
var user = db.Users.Where(x => x.Email == payload.Email).FirstOrDefault(); }
if (user != null)
{
return Ok(JWTGenerator(user));
}
else
{
return Unauthorized();
}
}
private dynamic JWTGenerator(User user) private dynamic JwtGenerator(User user)
{
var key = Encoding.ASCII.GetBytes(_configuration.GetValue<string>("Secret")!);
var expirationTime = DateTime.UtcNow.AddMinutes(5);
var tokenDescriptor = new SecurityTokenDescriptor
{ {
var key = Encoding.ASCII.GetBytes(configuration.GetValue<string>("Secret")!); Subject = new ClaimsIdentity(new[]
var expirationTime = DateTime.UtcNow.AddMinutes(5);
var tokenDescriptor = new SecurityTokenDescriptor
{ {
Subject = new ClaimsIdentity(new[]
{
new Claim("Id", Guid.NewGuid().ToString()), new Claim("Id", Guid.NewGuid().ToString()),
new Claim(JwtRegisteredClaimNames.Sub, user.Id.ToString()), new Claim(JwtRegisteredClaimNames.Sub, user.Id.ToString()),
new Claim(JwtRegisteredClaimNames.Jti, new Claim(JwtRegisteredClaimNames.Jti,
Guid.NewGuid().ToString()) Guid.NewGuid().ToString())
}), }),
Expires = expirationTime, Expires = expirationTime,
SigningCredentials = new SigningCredentials SigningCredentials = new SigningCredentials
(new SymmetricSecurityKey(key), (new SymmetricSecurityKey(key),
SecurityAlgorithms.HmacSha512Signature) SecurityAlgorithms.HmacSha512Signature)
}; };
var tokenHandler = new JwtSecurityTokenHandler(); var tokenHandler = new JwtSecurityTokenHandler();
var token = tokenHandler.CreateToken(tokenDescriptor); var token = tokenHandler.CreateToken(tokenDescriptor);
var jwtToken = tokenHandler.WriteToken(token); var stringToken = tokenHandler.WriteToken(token);
var stringToken = tokenHandler.WriteToken(token); return new { token = stringToken, id = user.Id, expirationTime };
return new { token = stringToken, id = user.Id, expirationTime };
}
} }
} }

File diff suppressed because it is too large Load Diff

View File

@@ -2,72 +2,58 @@ using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data; using Google.Apis.Sheets.v4.Data;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using WebAPI.Models; using WebAPI.Models;
namespace WebAPI.Controllers namespace WebAPI.Controllers;
public class LogsController : Controller
{ {
public class LogsController : Controller private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues;
private readonly IConfiguration _configuration;
public LogsController(
GoogleSheetsHelper googleSheetsHelper,
IConfiguration configuration)
{ {
private SpreadsheetsResource.ValuesResource? googleSheetValues; if (googleSheetsHelper.Service is not null) {
private GoogleDriveHelper googleDriveHelper; _googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values;
private readonly IConfiguration configuration; }
public LogsController( _configuration = configuration;
GoogleSheetsHelper _googleSheetsHelper, }
GoogleDriveHelper _googleDriveHelper,
IConfiguration _configuration) public void AddEntry(LogEntry entry)
{ {
if (_googleSheetsHelper.Service is not null) { if (_googleSheetValues is null) {
googleSheetValues = _googleSheetsHelper.Service.Spreadsheets.Values; throw new Exception("Google Sheets API not initialized");
}
googleDriveHelper = _googleDriveHelper;
configuration = _configuration;
} }
public void AddEntry(LogEntry entry) var type = entry.LogType switch
{ {
if (googleSheetValues is null) { LogType.import => "Import",
throw new Exception("Google Sheets API not initialized"); LogType.backup => "Backup",
} LogType.process => "Process",
String type; LogType.powerBI => "PowerBIAccess",
switch (entry.LogType) { _ => "Other"
case LogType.import: };
type = "Import"; var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();
break; var data = response.Values;
case LogType.backup: var row = 1;
type = "Backup"; if (data != null) {
break; row = data.Count + 1;
case LogType.process:
type = "Process";
break;
case LogType.powerBI:
type = "PowerBIAccess";
break;
default:
type = "Other"; // should never happen
break;
}
var response = googleSheetValues.Get(configuration["appLogsFile"], $"{type}!A:A").Execute();
var data = response.Values;
int row = 1;
if (data != null) {
row = data.Count + 1;
}
var range = $"{type}!A{row}:D{row}";
List<object> logRow = new List<object>
{
entry.CreatedAt.ToString(new CultureInfo("pl-PL")),
entry.Type.ToString(),
entry.Title!,
entry.Message!
};
ValueRange valueRange = new ValueRange() { Values = new IList<object>[] { logRow }};
var updateRequest = googleSheetValues.Update(valueRange, configuration["appLogsFile"], range);
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
updateRequest.Execute();
} }
var range = $"{type}!A{row}:D{row}";
var logRow = new List<object>
{
entry.CreatedAt.ToString(new CultureInfo("pl-PL")),
entry.Type.ToString(),
entry.Title!,
entry.Message!
};
var valueRange = new ValueRange { Values = new IList<object>[] { logRow }};
var updateRequest = _googleSheetValues.Update(valueRange, _configuration["appLogsFile"], range);
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
updateRequest.Execute();
} }
} }

View File

@@ -1,26 +1,25 @@
using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
namespace WebAPI.Controllers namespace WebAPI.Controllers;
{
[ApiController]
[Route("api/[controller]")]
[Authorize]
public class PingController : Controller
{
private readonly IConfiguration _configuration;
public PingController(
IConfiguration configuration)
{
_configuration = configuration;
}
[HttpGet] [ApiController]
[Route("Ping")] [Route("api/[controller]")]
[AllowAnonymous] [Authorize]
public IActionResult Ping() public class PingController : Controller
{ {
return Ok(_configuration["PONG"]); private readonly IConfiguration _configuration;
} public PingController(
IConfiguration configuration)
{
_configuration = configuration;
}
[HttpGet]
[Route("Ping")]
[AllowAnonymous]
public IActionResult Ping()
{
return Ok(_configuration["PONG"]);
} }
} }

View File

@@ -1,99 +1,121 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Drive.v3.Data;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data; using Google.Apis.Sheets.v4.Data;
using WebAPI.Models; using WebAPI.Models;
using static Google.Apis.Drive.v3.FilesResource;
namespace WebAPI.Exports namespace WebAPI.Exports;
public class GoogleSheetExport
{ {
public class googleSheetExport private readonly GoogleDriveHelper _googleDriveHelper;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly IConfiguration _configuration;
public GoogleSheetExport(
GoogleDriveHelper googleDriveHelper,
SpreadsheetsResource.ValuesResource googleSheetValues,
IConfiguration configuration)
{ {
private GoogleDriveHelper googleDriveHelper; _googleDriveHelper = googleDriveHelper;
private SpreadsheetsResource.ValuesResource googleSheetValues; _googleSheetValues = googleSheetValues;
private readonly IConfiguration configuration; _configuration = configuration;
public googleSheetExport( }
GoogleDriveHelper _googleDriveHelper, public void Export(Layer layer)
SpreadsheetsResource.ValuesResource _googleSheetValues, {
IConfiguration _configuration) if (_googleDriveHelper.Service is null)
{ {
googleDriveHelper = _googleDriveHelper; throw new Exception("Google Drive API not initialized");
googleSheetValues = _googleSheetValues;
configuration = _configuration;
} }
public void export(Layer layer) try
{ {
if (googleDriveHelper.Service is null)
{
throw new Exception("Google Drive API not initialized");
}
try
{
List<IList<object>> data = new List<IList<object>>() { new List<object>() { layer.Name! } }; var data = new List<IList<object>> { new List<object> { layer.Name! } };
switch (layer.Type) switch (layer.Type)
{
case LayerType.import:
{ {
case LayerType.import: data.Add(new List<object> { "Code", "Value1" });
{ data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Value1! }));
data.Add(new List<object> { "Code", "Value1" }); break;
foreach (Record record in layer.Records!)
{
data.Add(new List<object> { record.Code!, record.Value1! });
}
break;
}
case LayerType.administration:
{
data.Add(new List<object> { "Code", "Desc1"});
foreach (Record record in layer.Records!)
{
data.Add(new List<object> { record.Code!, record.Desc1!});
}
break;
}
case LayerType.processed:
{
data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3",
"Value5", "Value6", "Value7", "Value8", "Value9", "Value10",
"Value11", "Value12", "Value13", "Value14", "Value15", "Value16",
"Value17", "Value18", "Value19", "Value20", "Value21", "Value22",
"Value23", "Value24", "Value25", "Value26", "Value27", "Value28",
"Value29", "Value30", "Value31", "Value32"});
foreach (Record record in layer.Records!)
{
data.Add(new List<object> { record.Code!, record.Value1!, record.Value2!, record.Value3!, record.Value4!,
record.Value5!, record.Value6!, record.Value7!, record.Value8!, record.Value9!, record.Value10!,
record.Value11!, record.Value12!, record.Value13!, record.Value14!, record.Value15!, record.Value16!,
record.Value17!, record.Value18!, record.Value19!, record.Value20!, record.Value21!, record.Value22!,
record.Value23!, record.Value24!, record.Value25!, record.Value26!, record.Value27!, record.Value28!,
record.Value29!, record.Value30!, record.Value31!, record.Value32!});
}
break;
}
} }
case LayerType.administration:
{
data.Add(new List<object> { "Code", "Desc1"});
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Desc1! }));
break;
}
case LayerType.processed:
{
data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3",
"Value5", "Value6", "Value7", "Value8", "Value9", "Value10",
"Value11", "Value12", "Value13", "Value14", "Value15", "Value16",
"Value17", "Value18", "Value19", "Value20", "Value21", "Value22",
"Value23", "Value24", "Value25", "Value26", "Value27", "Value28",
"Value29", "Value30", "Value31", "Value32"});
Google.Apis.Drive.v3.Data.File body = new Google.Apis.Drive.v3.Data.File(); data.AddRange(layer.Records!.Select(record => new List<object>
body.Name = $"{DateTime.Now.ToString(new CultureInfo("pl-PL"))}"; {
body.MimeType = "application/vnd.google-apps.spreadsheet"; record.Code!,
body.Parents = new List<string?> { configuration["exportDirectory"] }; record.Value1!,
CreateRequest request = googleDriveHelper.Service.Files.Create(body); record.Value2!,
var file = request.Execute(); record.Value3!,
record.Value4!,
string sheetId = file.Id; record.Value5!,
var range = $"Sheet1!A1:AG${data.Count}"; record.Value6!,
record.Value7!,
ValueRange valueRange = new ValueRange() { Values = data}; record.Value8!,
record.Value9!,
var updateRequest = googleSheetValues.Update(valueRange, sheetId, range); record.Value10!,
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW; record.Value11!,
updateRequest.Execute(); record.Value12!,
record.Value13!,
} catch (Exception e) record.Value14!,
{ record.Value15!,
Console.WriteLine(e.ToString()); record.Value16!,
record.Value17!,
record.Value18!,
record.Value19!,
record.Value20!,
record.Value21!,
record.Value22!,
record.Value23!,
record.Value24!,
record.Value25!,
record.Value26!,
record.Value27!,
record.Value28!,
record.Value29!,
record.Value30!,
record.Value31!,
record.Value32!
}));
break;
}
default:
throw new Exception("Wrong LayerType");
} }
var body = new Google.Apis.Drive.v3.Data.File
{
Name = $"{DateTime.Now.ToString(new CultureInfo("pl-PL"))}",
MimeType = "application/vnd.google-apps.spreadsheet",
Parents = new List<string?> { _configuration["exportDirectory"] }
};
var request = _googleDriveHelper.Service.Files.Create(body);
var file = request.Execute();
var sheetId = file.Id;
var range = $"Sheet1!A1:AG${data.Count}";
var valueRange = new ValueRange { Values = data};
var updateRequest = _googleSheetValues.Update(valueRange, sheetId, range);
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
updateRequest.Execute();
} catch (Exception e)
{
Console.WriteLine(e.ToString());
} }
} }
} }

View File

@@ -6,7 +6,7 @@ namespace WebAPI;
public class GoogleDriveHelper public class GoogleDriveHelper
{ {
public DriveService? Service { get; set; } public DriveService? Service { get; private set; }
private const string ApplicationName = "Diuna"; private const string ApplicationName = "Diuna";
private static readonly string[] Scopes = { DriveService.Scope.Drive }; private static readonly string[] Scopes = { DriveService.Scope.Drive };
public GoogleDriveHelper() public GoogleDriveHelper()

View File

@@ -1,64 +1,70 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models namespace WebAPI.Models;
public class Record
{ {
public class Record #region Properties
{ [Key]
#region Properties public Guid Id { get; set; }
[Key] [Required]
public Guid Id { get; set; } [StringLength(50)]
[Required] public string? Code { get; set; }
public string? Code { get; set; } public double? Value1 { get; set; }
public double? Value1 { get; set; } public double? Value2 { get; set; }
public double? Value2 { get; set; } public double? Value3 { get; set; }
public double? Value3 { get; set; } public double? Value4 { get; set; }
public double? Value4 { get; set; } public double? Value5 { get; set; }
public double? Value5 { get; set; } public double? Value6 { get; set; }
public double? Value6 { get; set; } public double? Value7 { get; set; }
public double? Value7 { get; set; } public double? Value8 { get; set; }
public double? Value8 { get; set; } public double? Value9 { get; set; }
public double? Value9 { get; set; } public double? Value10 { get; set; }
public double? Value10 { get; set; } public double? Value11 { get; set; }
public double? Value11 { get; set; } public double? Value12 { get; set; }
public double? Value12 { get; set; } public double? Value13 { get; set; }
public double? Value13 { get; set; } public double? Value14 { get; set; }
public double? Value14 { get; set; } public double? Value15 { get; set; }
public double? Value15 { get; set; } public double? Value16 { get; set; }
public double? Value16 { get; set; } public double? Value17 { get; set; }
public double? Value17 { get; set; } public double? Value18 { get; set; }
public double? Value18 { get; set; } public double? Value19 { get; set; }
public double? Value19 { get; set; } public double? Value20 { get; set; }
public double? Value20 { get; set; } public double? Value21 { get; set; }
public double? Value21 { get; set; } public double? Value22 { get; set; }
public double? Value22 { get; set; } public double? Value23 { get; set; }
public double? Value23 { get; set; } public double? Value24 { get; set; }
public double? Value24 { get; set; } public double? Value25 { get; set; }
public double? Value25 { get; set; } public double? Value26 { get; set; }
public double? Value26 { get; set; } public double? Value27 { get; set; }
public double? Value27 { get; set; } public double? Value28 { get; set; }
public double? Value28 { get; set; } public double? Value29 { get; set; }
public double? Value29 { get; set; } public double? Value30 { get; set; }
public double? Value30 { get; set; } public double? Value31 { get; set; }
public double? Value31 { get; set; } public double? Value32 { get; set; }
public double? Value32 { get; set; } //Description fields
//Description fields [StringLength(50)]
public string? Desc1 { get; set; } public string? Desc1 { get; set; }
public string? Desc2 { get; set; } [StringLength(50)]
public string? Desc3 { get; set; } public string? Desc2 { get; set; }
public string? Desc4 { get; set; } [StringLength(50)]
public string? Desc5 { get; set; } public string? Desc3 { get; set; }
public DateTime CreatedAt { get; set; } [StringLength(50)]
public DateTime ModifiedAt { get; set; } public string? Desc4 { get; set; }
public bool IsDeleted { get; set; } [StringLength(50)]
#endregion public string? Desc5 { get; set; }
#region Relations [StringLength(50)]
[Required] public DateTime CreatedAt { get; set; }
public Guid CreatedById { get; set; } public DateTime ModifiedAt { get; set; }
public User? CreatedBy { get; set; } public bool IsDeleted { get; init; }
[Required] #endregion
public Guid ModifiedById { get; set; } #region Relations
public User? ModifiedBy { get; set; } [Required]
public Guid LayerId { get; set; } public Guid CreatedById { get; set; }
#endregion public User? CreatedBy { get; init; }
} [Required]
} public Guid ModifiedById { get; set; }
public User? ModifiedBy { get; init; }
public Guid LayerId { get; set; }
#endregion
}

View File

@@ -1,16 +1,16 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace WebAPI.Models namespace WebAPI.Models;
public class User
{ {
public class User #region Properties
{ [Key]
#region Properties public Guid Id { get; init; }
[Key] [StringLength(50)]
public Guid Id { get; set; } public string? Email { get; init; }
public string? Email { get; set; } [StringLength(50)]
[StringLength(50)] public string? UserName { get; init; }
public string? UserName { get; set; } public DateTime CreatedAt { get; init; }
public DateTime CreatedAt { get; set; } #endregion
#endregion }
}
}

View File

@@ -1,136 +1,127 @@
using System; using System.Globalization;
using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using WebAPI; using WebAPI;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace DiunaBIWebAPI.dataImporters namespace DiunaBIWebAPI.dataImporters;
public class MorskaFk2Importer
{ {
public class MorskaFk2Importer private readonly AppDbContext _db;
{ private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly AppDbContext db; private readonly LayersController _controller;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public MorskaFk2Importer( public MorskaFk2Importer(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController _controller) LayersController controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _googleSheetValues = googleSheetValues;
controller = _controller; _controller = controller;
} }
public void import(Layer importWorker) public void Import(Layer importWorker)
{ {
string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
var importYearCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
var importMonthCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
var importNameCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {importWorker.Name}");
}
var checkSumCell = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (checkSumCell == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
if (dataRange == null)
{
throw new Exception($"DataRange not found, {importWorker.Name}");
}
var nameResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
var name = nameResponse.Values?[0][0].ToString();
if (name == null)
{
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
}
var yearResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
var year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
var month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
var layer = new Layer
{
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}2-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (var i = 0; i < data.Count; i++)
{
if (data[i].Count <= 8 || (string)data[i][2] == string.Empty) continue;
var dateArr = data[i][0].ToString()!.Split(".");
if (dateArr.Length != 3)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"Invalid date in row {i}");
}
string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1;
if (sheetTabName == null)
{
throw new Exception($"SheetTabName not found, {importWorker.Name}");
}
string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1;
if (importYearCell == null)
{
throw new Exception($"ImportYear not found, {importWorker.Name}");
}
string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1;
if (importMonthCell == null)
{
throw new Exception($"ImportMonth not found, {importWorker.Name}");
}
string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1;
if (importNameCell == null)
{
throw new Exception($"ImportName not found, {importWorker.Name}");
}
string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1;
if (checkSumCell == null)
{
throw new Exception($"SheetId not found, {importWorker.Name}");
}
string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1;
if (dataRange == null)
{
throw new Exception($"DataRange not found, {importWorker.Name}");
} }
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); var number = data[i][1].ToString()!;
string? name = nameResponse.Values?[0][0].ToString(); if (number.Length == 1) number = $"0{number}";
if (name == null) var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (!(data[i][8].ToString()?.Length > 0) ||
!double.TryParse(data[i][8].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
var record = new Record
{ {
throw new Exception($"ImportName cell is empty, {importWorker.Name}"); Id = Guid.NewGuid(),
} Code = code,
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); Desc1 = data[i][2].ToString(),
string? year = yearResponse.Values[0][0].ToString(); Value1 = value,
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
Layer layer = new Layer
{
Source = "GoogleSheet",
Number = db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}2-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; newRecords.Add(record);
List<Record> newRecords = new List<Record>();
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (int i = 0; i < data.Count; i++)
{
if (data[i].Count > 8 && (string)data[i][2] != String.Empty)
{
string[] dateArr = data[i][0].ToString()!.Split(".");
if (dateArr.Length != 3)
{
throw new Exception($"Invalid date in row {i}");
}
string number = data[i][1].ToString()!;
if (number.Length == 1) number = $"0{number}";
string code = dateArr[2] + dateArr[1] + dateArr[0] + number;
double value;
if (
data[i][8].ToString()?.Length > 0 &&
double.TryParse(data[i][8].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
{
Record record = new Record
{
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][2].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
};
}
}
db.Layers.Add(layer);
controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
} _db.Layers.Add(layer);
} _controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

View File

@@ -1,123 +1,116 @@
using System; using System.Globalization;
using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using WebAPI; using WebAPI;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
namespace DiunaBIWebAPI.dataImporters namespace DiunaBIWebAPI.dataImporters;
public class MorskaImporter
{ {
public class MorskaImporter private readonly AppDbContext _db;
{ private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly AppDbContext db; private readonly LayersController _controller;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues;
private readonly LayersController controller;
public MorskaImporter( public MorskaImporter(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController _controller) LayersController controller)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _googleSheetValues = googleSheetValues;
controller = _controller; _controller = controller;
} }
public void import(Layer importWorker) public void Import(Layer importWorker)
{ {
string? sheetId = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
string? sheetTabName = importWorker.Records!.Where(x => x.Code == "SheetTabName").FirstOrDefault()?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
string? importYearCell = importWorker.Records!.Where(x => x.Code == "ImportYear").FirstOrDefault()?.Desc1; var importYearCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (importYearCell == null) if (importYearCell == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
string? importMonthCell = importWorker.Records!.Where(x => x.Code == "ImportMonth").FirstOrDefault()?.Desc1; var importMonthCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (importMonthCell == null) if (importMonthCell == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
string? importNameCell = importWorker.Records!.Where(x => x.Code == "ImportName").FirstOrDefault()?.Desc1; var importNameCell = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (importNameCell == null) if (importNameCell == null)
{ {
throw new Exception($"ImportName not found, {importWorker.Name}"); throw new Exception($"ImportName not found, {importWorker.Name}");
} }
string? checkSumCell = importWorker.Records!.Where(x => x.Code == "SheetId").FirstOrDefault()?.Desc1; var checkSumCell = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (checkSumCell == null) if (checkSumCell == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
string? dataRange = importWorker.Records!.Where(x => x.Code == "DataRange").FirstOrDefault()?.Desc1; var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
if (dataRange == null) if (dataRange == null)
{ {
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
var nameResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute(); var nameResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importNameCell}:{importNameCell}").Execute();
string? name = nameResponse.Values?[0][0].ToString(); var name = nameResponse.Values?[0][0].ToString();
if (name == null) if (name == null)
{
throw new Exception($"ImportName cell is empty, {importWorker.Name}");
}
var yearResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute();
var year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
var month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
var layer = new Layer
{
Source = "GoogleSheet",
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (var i = 0; i < data[1].Count; i++)
{
if (!(data[0][i].ToString()?.Length > 0) ||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
var record = new Record
{ {
throw new Exception($"ImportName cell is empty, {importWorker.Name}"); Id = Guid.NewGuid(),
} Code = data[0][i].ToString(),
var yearResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importYearCell}:{importYearCell}").Execute(); Value1 = value,
string? year = yearResponse.Values[0][0].ToString();
if (year == null)
{
throw new Exception($"ImportYear cell is empty, {importWorker.Name}");
}
var monthResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{importMonthCell}:{importMonthCell}").Execute();
string? month = monthResponse.Values[0][0].ToString();
if (month == null)
{
throw new Exception($"ImportMonth cell is empty, {importWorker.Name}");
}
Layer layer = new Layer
{
Source = "GoogleSheet",
Number = db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm")}"; newRecords.Add(record);
List<Record> newRecords = new List<Record>();
var dataRangeResponse = googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
for (int i = 0; i < data[1].Count; i++)
{
double value;
if (
data[0][i].ToString()?.Length > 0 &&
double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
{
Record record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
};
}
db.Layers.Add(layer);
controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges();
} }
} _db.Layers.Add(layer);
} _controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges();
}
}

View File

@@ -1,53 +0,0 @@
using CsvHelper;
using CsvHelper.Configuration;
using Microsoft.EntityFrameworkCore.Metadata.Internal;
using System.Formats.Asn1;
using System.Globalization;
using System.Text;
using WebAPI.Models;
namespace WebAPI.dataParsers
{
public class csvParser
{
public List<Record> parse(IFormFile file)
{
var info = CultureInfo.CurrentCulture;
List<Record> records = new List<Record>();
var stream = new StreamReader(file.OpenReadStream());
string content = stream.ReadToEnd();
List<string> lines = content.Split("\n").ToList();
List<List<string>> data = new List<List<string>>();
foreach (string line in lines)
{
data.Add(line.Split(";").ToList());
}
for (int i = 1; i < data[0].Count; i++)
{
for (int j = 1; j < data.Count; j++) {
if (data[j][0].Length > 0)
{
double value = double.Parse(data[j][i], CultureInfo.GetCultureInfo("pl-PL"));
if (value > 0)
{
Record record = new Record();
record.Id = Guid.NewGuid();
record.Code = data[0][i];
record.Desc1 = data[j][0];
record.Value1 = value;
record.CreatedAt = DateTime.UtcNow;
record.ModifiedAt= DateTime.UtcNow;
records.Add(record);
}
}
};
}
return records;
}
}
}

View File

@@ -1,62 +0,0 @@
using Google.Apis.Sheets.v4;
using System.Globalization;
using WebAPI.Models;
namespace WebAPI.dataParsers
{
public class morskaK5Parser
{
private SpreadsheetsResource.ValuesResource googleSheetValues;
private AppDbContext db;
public morskaK5Parser(
SpreadsheetsResource.ValuesResource _googleSheetValues,
AppDbContext _db)
{
googleSheetValues = _googleSheetValues;
db = _db;
}
public Layer parse()
{
Layer layer = new Layer();
layer.Source = "GoogleSheet";
string sheetId = "1ZzndU8HjYqz5VKCcrVHBOFW8fqpYfwquclznX9q39Yk";
var range = "Sierpien_2023!B3:AR5";
var request = googleSheetValues.Get(sheetId, range);
var response = request.Execute();
var data = response.Values;
layer.Source = "GoogleSheet";
layer.Number = db.Layers.Count() + 1;
layer.Name = $"L{layer.Number}-I-{data[0][1]}-{data[0][2]}/{data[0][3]}-{DateTime.Now.ToString("yyyyMMddHHmm")}";
layer.Type = LayerType.import;
List<Record> records = new List<Record>();
for (int i = 1; i < data[1].Count; i++)
{
double value;
if (
data[1][i].ToString()?.Length > 0 &&
double.TryParse(data[2][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out value))
{
Record record = new Record();
record.Id = Guid.NewGuid();
record.Code = data[1][i].ToString();
record.Value1 = value;
record.CreatedAt = DateTime.UtcNow;
record.ModifiedAt = DateTime.UtcNow;
records.Add(record);
};
}
layer.Records = records;
return layer;
}
}
}

View File

@@ -1,209 +1,175 @@
using System; using WebAPI.Models;
using WebAPI.Models;
namespace DiunaBIWebAPI.dataProcessors namespace DiunaBIWebAPI.dataProcessors;
public static class ProcessHelper
{ {
public static class ProcessHelper public static void SetValue(Record record, int number, double? value)
{ {
public static void setValue(Record record, int number, double? value) value = (double)Math.Round((decimal)(value ?? 0), 2);
switch (number)
{ {
value = (double)Math.Round((decimal)(value ?? 0), 2); case 1:
switch (number) record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
public static double? getValue(Record record, int number)
{
return number switch
{
1 => record.Value1,
2 => record.Value2,
3 => record.Value3,
4 => record.Value4,
5 => record.Value5,
6 => record.Value6,
7 => record.Value7,
8 => record.Value8,
9 => record.Value9,
10 => record.Value10,
11 => record.Value11,
12 => record.Value12,
13 => record.Value13,
14 => record.Value14,
15 => record.Value15,
16 => record.Value16,
17 => record.Value17,
18 => record.Value18,
19 => record.Value19,
20 => record.Value20,
21 => record.Value21,
22 => record.Value22,
23 => record.Value23,
24 => record.Value24,
25 => record.Value25,
26 => record.Value26,
27 => record.Value27,
28 => record.Value28,
29 => record.Value29,
30 => record.Value30,
31 => record.Value31,
32 => record.Value32,
_ => null
};
}
public static List<int> ParseCodes(string codes)
{
var codesList = new List<int>();
foreach (var code in codes.Split(';'))
{
var range = code.Split('-');
switch (range.Length)
{ {
case 1: case 1:
record.Value1 = value;
break;
case 2:
record.Value2 = value;
break;
case 3:
record.Value3 = value;
break;
case 4:
record.Value4 = value;
break;
case 5:
record.Value5 = value;
break;
case 6:
record.Value6 = value;
break;
case 7:
record.Value7 = value;
break;
case 8:
record.Value8 = value;
break;
case 9:
record.Value9 = value;
break;
case 10:
record.Value10 = value;
break;
case 11:
record.Value11 = value;
break;
case 12:
record.Value12 = value;
break;
case 13:
record.Value13 = value;
break;
case 14:
record.Value14 = value;
break;
case 15:
record.Value15 = value;
break;
case 16:
record.Value16 = value;
break;
case 17:
record.Value17 = value;
break;
case 18:
record.Value18 = value;
break;
case 19:
record.Value19 = value;
break;
case 20:
record.Value20 = value;
break;
case 21:
record.Value21 = value;
break;
case 22:
record.Value22 = value;
break;
case 23:
record.Value23 = value;
break;
case 24:
record.Value24 = value;
break;
case 25:
record.Value25 = value;
break;
case 26:
record.Value26 = value;
break;
case 27:
record.Value27 = value;
break;
case 28:
record.Value28 = value;
break;
case 29:
record.Value29 = value;
break;
case 30:
record.Value30 = value;
break;
case 31:
record.Value31 = value;
break;
case 32:
record.Value32 = value;
break;
}
}
public static double? getValue(Record record, int number)
{
switch (number)
{
case 1:
return record.Value1;
case 2:
return record.Value2;
case 3:
return record.Value3;
case 4:
return record.Value4;
case 5:
return record.Value5;
case 6:
return record.Value6;
case 7:
return record.Value7;
case 8:
return record.Value8;
case 9:
return record.Value9;
case 10:
return record.Value10;
case 11:
return record.Value11;
case 12:
return record.Value12;
case 13:
return record.Value13;
case 14:
return record.Value14;
case 15:
return record.Value15;
case 16:
return record.Value16;
case 17:
return record.Value17;
case 18:
return record.Value18;
case 19:
return record.Value19;
case 20:
return record.Value20;
case 21:
return record.Value21;
case 22:
return record.Value22;
case 23:
return record.Value23;
case 24:
return record.Value24;
case 25:
return record.Value25;
case 26:
return record.Value26;
case 27:
return record.Value27;
case 28:
return record.Value28;
case 29:
return record.Value29;
case 30:
return record.Value30;
case 31:
return record.Value31;
case 32:
return record.Value32;
default:
return null;
}
}
public static List<int> parseCodes(string codes)
{
List<int> codesList = new List<int>();
foreach (string code in codes.Split(';'))
{
string[] range = code.Split('-');
if (range.Length == 1)
{
codesList.Add(int.Parse(range[0])); codesList.Add(int.Parse(range[0]));
} break;
else if (range.Length == 2) case 2:
{ {
for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++) for (int i = int.Parse(range[0]); i <= int.Parse(range[1]); i++)
{ {
codesList.Add(i); codesList.Add(i);
} }
}
else
{
throw new Exception($"Invalid code range: {code}");
}
}
return codesList;
}
}
}
break;
}
default:
throw new Exception($"Invalid code range: {code}");
}
}
return codesList;
}
}

View File

@@ -40,7 +40,7 @@ namespace WebAPI.dataProcessors
{ {
throw new Exception("Codes record not found"); throw new Exception("Codes record not found");
} }
List<int> codesList = ProcessHelper.parseCodes(codes); List<int> codesList = ProcessHelper.ParseCodes(codes);
Layer? processedLayer = db.Layers Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker!.Id

View File

@@ -38,7 +38,7 @@ namespace WebAPI.dataProcessors
throw new Exception("Codes record not found"); throw new Exception("Codes record not found");
} }
// create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512] // create array of integers from string codes where: '501-503;505-505;510-512' -> [501,502,503,505,510,511,512]
List<int> codesList = ProcessHelper.parseCodes(codes); List<int> codesList = ProcessHelper.ParseCodes(codes);
Layer? processedLayer = db.Layers Layer? processedLayer = db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker!.Id
@@ -103,7 +103,7 @@ namespace WebAPI.dataProcessors
}; };
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.setValue(newRecord, i, ProcessHelper.getValue(x, i)); ProcessHelper.SetValue(newRecord, i, ProcessHelper.getValue(x, i));
} }
return newRecord; return newRecord;
}) })

View File

@@ -93,7 +93,7 @@ namespace WebAPI.dataProcessors
}; };
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.setValue(processedRecord, i, ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i))); codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
} }

View File

@@ -106,7 +106,7 @@ namespace WebAPI.dataProcessors
}; };
for (var i = 1; i<33; i++) for (var i = 1; i<33; i++)
{ {
ProcessHelper.setValue(processedRecord, i, ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i))); codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
} }
newRecords.Add(processedRecord); newRecords.Add(processedRecord);

View File

@@ -128,10 +128,10 @@ namespace WebAPI.dataProcessors
Record validationRecord = new Record(); Record validationRecord = new Record();
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.setValue(processedRecord, i, ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i))); codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
ProcessHelper.setValue(validationRecord, i, ProcessHelper.SetValue(validationRecord, i,
codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i))); codeRecordsValidation.Sum(x => ProcessHelper.getValue(x, i)));
if ( if (

View File

@@ -108,7 +108,7 @@ namespace WebAPI.dataProcessors
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)) .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1))
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1 ?? 0; .FirstOrDefault()?.Value1 ?? 0;
ProcessHelper.setValue(processedRecord, 1, firstVal); ProcessHelper.SetValue(processedRecord, 1, firstVal);
previousValue = firstVal; previousValue = firstVal;
//days 2-29/30 //days 2-29/30
for (int i=2; i<lastDayInMonth; i++) for (int i=2; i<lastDayInMonth; i++)
@@ -120,11 +120,11 @@ namespace WebAPI.dataProcessors
if (dayVal == null) if (dayVal == null)
{ {
//TODO: missing day value? Should I log it? //TODO: missing day value? Should I log it?
ProcessHelper.setValue(processedRecord, i, 0); ProcessHelper.SetValue(processedRecord, i, 0);
} else } else
{ {
double processedVal = (dayVal ?? 0) - previousValue; double processedVal = (dayVal ?? 0) - previousValue;
ProcessHelper.setValue(processedRecord, i, processedVal); ProcessHelper.SetValue(processedRecord, i, processedVal);
previousValue = dayVal ?? 0; previousValue = dayVal ?? 0;
} }
} }
@@ -136,17 +136,17 @@ namespace WebAPI.dataProcessors
if (lastVal == null) if (lastVal == null)
{ {
ProcessHelper.setValue(processedRecord, lastDayInMonth, 0); ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
} else } else
{ {
ProcessHelper.setValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue); ProcessHelper.SetValue(processedRecord, lastDayInMonth, (lastVal ?? 0) - previousValue);
} }
// copy last value // copy last value
double? valueToCopy = codeRecords double? valueToCopy = codeRecords
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault()?.Value1; .FirstOrDefault()?.Value1;
ProcessHelper.setValue(processedRecord, 32, valueToCopy); ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord); newRecords.Add(processedRecord);
} }

View File

@@ -95,7 +95,7 @@ namespace WebAPI.dataProcessors
}; };
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.setValue(processedRecord, i, ProcessHelper.SetValue(processedRecord, i,
codeRecords.Sum(x => ProcessHelper.getValue(x, i))); codeRecords.Sum(x => ProcessHelper.getValue(x, i)));
} }
newRecords.Add(processedRecord); newRecords.Add(processedRecord);

View File

@@ -1,8 +1,5 @@
using System.Globalization; using DiunaBIWebAPI.dataProcessors;
using DiunaBIWebAPI.dataProcessors;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.AspNetCore.Http.HttpResults;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using WebAPI.Controllers; using WebAPI.Controllers;
using WebAPI.Models; using WebAPI.Models;
@@ -12,46 +9,44 @@ namespace WebAPI.dataProcessors
{ {
public class T4R2Processor public class T4R2Processor
{ {
private readonly AppDbContext db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource googleSheetValues; private readonly LayersController _controller;
private readonly LayersController controller; private readonly LogsController _logsController;
private readonly LogsController logsController;
public T4R2Processor( public T4R2Processor(
AppDbContext _db, AppDbContext db,
SpreadsheetsResource.ValuesResource _googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
LayersController _controller, LayersController controller,
LogsController _logsController) LogsController logsController)
{ {
db = _db; _db = db;
googleSheetValues = _googleSheetValues; _controller = controller;
controller = _controller; _logsController = logsController;
logsController = _logsController;
} }
public void process(Layer processWorker) public void Process(Layer processWorker)
{ {
int year = int.Parse(processWorker!.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
List<Record>? sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count() == 0) if (!sources!.Any())
{ {
throw new Exception("Source record not found"); throw new Exception("Source record not found");
} }
string? layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1;
if (layerName == null) if (layerName == null)
{ {
throw new Exception("LayerName record not found"); throw new Exception("LayerName record not found");
} }
Layer? processedLayer = db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker!.Id .Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted) && !x.IsDeleted)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
bool isNew = false; var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true; isNew = true;
@@ -60,8 +55,8 @@ namespace WebAPI.dataProcessors
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Source = "", Source = "",
Type = LayerType.processed, Type = LayerType.processed,
ParentId = processWorker!.Id, ParentId = processWorker.Id,
Number = db.Layers.Count() + 1, Number = _db.Layers.Count() + 1
}; };
processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; processedLayer.Name = $"L{processedLayer.Number}-{layerName}";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -75,42 +70,35 @@ namespace WebAPI.dataProcessors
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
List<Record> newRecords = new List<Record>(); var newRecords = new List<Record>();
foreach (Record source in sources!) foreach (var source in sources!)
{ {
string? rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}")
?.Desc1; ?.Desc1;
List<int> sourceCodes = new List<int>(); var sourceCodes = new List<int>();
if (rawSourceCodes != null) if (rawSourceCodes != null)
{ {
sourceCodes = ProcessHelper.parseCodes(rawSourceCodes); sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes);
} }
for (int month = 1; month <= DateTime.UtcNow.Month; month++) for (var month = 1; month <= DateTime.UtcNow.Month; month++)
{ {
Layer? dataSource = db.Layers.Where(x => var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/{month}-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{year}/{monthCopy}-{source.Desc1}-T")
) )
.Include(x => x.Records) .Include(x => x.Records)
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
List<Record> news = dataSource.Records! List<Record> news = dataSource.Records!
.Where(x => .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
{
if (sourceCodes.Count > 0)
{
return sourceCodes.Contains(int.Parse(x.Code!));
}
return true; // get all
})
.Select(x => .Select(x =>
{ {
Record newRecord = new Record var newRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = x.Code + month.ToString("D2"), Code = x.Code + month.ToString("D2"),
@@ -126,9 +114,9 @@ namespace WebAPI.dataProcessors
} }
else else
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.warning,
LogType = LogType.process, LogType = LogType.process,
Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found", Message = $"Data source {year}/{month}-{source.Desc1}-T3 not found",
@@ -138,7 +126,7 @@ namespace WebAPI.dataProcessors
} }
// year summery // year summery
Layer? dataSourceSum = db.Layers.Where(x => var dataSourceSum = _db.Layers.Where(x =>
x.Type == LayerType.processed && x.Type == LayerType.processed &&
!x.IsDeleted && !x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T")
@@ -147,7 +135,7 @@ namespace WebAPI.dataProcessors
.FirstOrDefault(); .FirstOrDefault();
if (dataSourceSum != null) if (dataSourceSum != null)
{ {
List<Record> news = dataSourceSum.Records! var news = dataSourceSum.Records!
.Where(x => .Where(x =>
{ {
if (sourceCodes.Count > 0) if (sourceCodes.Count > 0)
@@ -174,9 +162,9 @@ namespace WebAPI.dataProcessors
} }
else else
{ {
logsController.AddEntry(new LogEntry _logsController.AddEntry(new LogEntry
{ {
Title = $"{processWorker!.Name}, {processWorker.Id}", Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.warning, Type = LogEntryType.warning,
LogType = LogType.process, LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found", Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
@@ -184,75 +172,18 @@ namespace WebAPI.dataProcessors
}); });
} }
} }
/*
// year summary
Layer? dataSourceSum = db.Layers.Where(x =>
x.Type == LayerType.processed &&
!x.IsDeleted &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")
)
.Include(x => x.Records)
.FirstOrDefault();
if (dataSourceSum != null)
{
dataSources.Add(dataSourceSum);
}
else
{
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker!.Name}, {processWorker.Id}",
Type = LogEntryType.warning,
LogType = LogType.process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
}
}
/*
if (dataSources.Count == 0)
{
throw new Exception($"DataSources are empty");
}
List<Record> newRecords = dataSources
.SelectMany(x => x.Records!)
.Where(x => codesList.Contains(int.Parse(x.Code!)))
.Select(x =>
{
Layer? layer = dataSources.SingleOrDefault(y => y.Id == x.LayerId);
string postFix = layer!.Name!.Split("/")[1].Split("-")[0];
if (postFix.Length == 1)
{
postFix = "0" + postFix;
}
Record newRecord = new Record
{
Id = Guid.NewGuid(),
Code = x.Code + postFix,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
};
return newRecord;
})
.ToList();
*/
if (isNew) if (isNew)
{ {
db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
} }
else else
{ {
db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
} }
controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
db.SaveChanges(); _db.SaveChanges();
} }
} }
} }