imports: save records

This commit is contained in:
Michał Zieliński
2025-06-02 20:11:29 +02:00
parent 5046a29ee1
commit 8efd1edfad
6 changed files with 366 additions and 119 deletions

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD1Importer> _logger;
public MorskaD1Importer( public MorskaD1Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaD1Importer> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -64,38 +79,74 @@ public class MorskaD1Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); try
var data = dataRangeResponse.Values; {
var newRecords = (from t in data var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
where t.Count > 1 && (string)t[0] != string.Empty var data = dataRangeResponse.Values;
select new Record
{ _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
Id = Guid.NewGuid(),
Code = t[0].ToString(), var newRecords = (from t in data
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null, where t.Count > 1 && (string)t[0] != string.Empty
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null, select new Record
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null, {
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null, Id = Guid.NewGuid(),
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null, Code = t[0].ToString(),
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null, Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null, Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null, Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null, Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null, Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null, Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null, Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null, Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null, Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null, Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
CreatedAt = DateTime.UtcNow, Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
ModifiedAt = DateTime.UtcNow Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
}).ToList(); Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
_db.Layers.Add(layer); Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
// TODO: Save records to the layer Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); CreatedAt = DateTime.UtcNow,
_db.SaveChanges(); ModifiedAt = DateTime.UtcNow
}).ToList();
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
private double? ParseValue(string? value) private double? ParseValue(string? value)
@@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result; return result;
} }
catch (FormatException) catch (FormatException e)
{ {
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
return null; return null;
} }
} }
private bool IndexExists(IList<object> array, int index) private bool IndexExists(IList<object> array, int index)
{ {
return array != null && index >= 0 && index < array.Count; return array != null && index >= 0 && index < array.Count;

View File

@@ -1,6 +1,7 @@
using System.Globalization; using System.Globalization;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -9,74 +10,136 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaD3Importer : MorskaBaseImporter public class MorskaD3Importer : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaD3"; public override string ImporterType => "MorskaD3";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<MorskaD3Importer> _logger;
public MorskaD3Importer( public MorskaD3Importer(
AppDbContext db) AppDbContext db,
ILogger<MorskaD3Importer> logger)
{ {
_db = db; _db = db;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
throw new Exception($"ImportName not found, {importWorker.Name}"); throw new Exception($"ImportName not found, {importWorker.Name}");
} }
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
if (name == null) if (type == null)
{ {
throw new Exception($"ImportType not found, {importWorker.Name}"); throw new Exception($"ImportType not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
if (dataInbox == null) if (dataInbox == null)
{ {
throw new Exception($"DataInbox not found, {type}"); throw new Exception($"DataInbox not found, {type}");
} }
var data = Convert.FromBase64String(dataInbox.Data);
var tst = Encoding.UTF8.GetString(data);
var records = JsonSerializer.Deserialize<List<Record>>(Encoding.UTF8.GetString(data));
if (records == null)
{
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
}
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
if (records.Count == 0)
{
throw new Exception($"No records found for {year}{month}");
}
records = records.Select(x =>
{
x.Id = Guid.NewGuid();
x.CreatedAt = DateTime.UtcNow;
x.ModifiedAt = DateTime.UtcNow;
return x;
}).ToList();
var layer = new Layer
{
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}";
_db.Layers.Add(layer); _logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
// TODO: Save records to the layer dataInbox.Id, dataInbox.CreatedAt);
//controller.SaveRecords(layer.Id, records, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
_db.SaveChanges(); try
{
var data = Convert.FromBase64String(dataInbox.Data);
var jsonString = Encoding.UTF8.GetString(data);
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
if (records == null)
{
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
}
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
if (records.Count == 0)
{
throw new Exception($"No records found for {year}{month}");
}
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
records.Count, year, month);
records = records.Select(x =>
{
x.Id = Guid.NewGuid();
x.CreatedAt = DateTime.UtcNow;
x.ModifiedAt = DateTime.UtcNow;
return x;
}).ToList();
var layer = new Layer
{
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id,
Type = LayerType.Import,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
_db.Layers.Add(layer);
SaveRecords(layer.Id, records);
_db.SaveChanges();
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
records.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaFk2Importer : MorskaBaseImporter public class MorskaFk2Importer : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaFK2"; public override string ImporterType => "MorskaFK2";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaFk2Importer> _logger;
public MorskaFk2Importer( public MorskaFk2Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaFk2Importer> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -52,8 +65,12 @@ public class MorskaFk2Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -62,40 +79,91 @@ public class MorskaFk2Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>(); var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); try
var data = dataRangeResponse.Values;
for (var i = 0; i < data.Count; i++)
{ {
if (data[i].Count <= 9 || (string)data[i][3] == string.Empty) continue; var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var dateArr = data[i][1].ToString()!.Split("."); var data = dataRangeResponse.Values;
if (dateArr.Length != 3)
_logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null)
{ {
throw new Exception($"Invalid date in row {i}"); for (var i = 0; i < data.Count; i++)
{
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i);
continue;
}
var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3)
{
_logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]);
throw new Exception($"Invalid date in row {i}");
}
var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i);
continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][3].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
} }
var number = data[i][2].ToString()!; _db.Layers.Add(layer);
if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number; SaveRecords(layer.Id, newRecords);
if (!(data[i][9].ToString()?.Length > 0) || _db.SaveChanges();
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue;
var record = new Record _logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
{ newRecords.Count, layer.Name, layer.Id);
Id = Guid.NewGuid(),
Code = code,
Desc1 = data[i][3].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
} }
_db.Layers.Add(layer); catch (Exception e)
// TODO: Save records to the layer {
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId);
_db.SaveChanges(); throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaImporter : MorskaBaseImporter public class MorskaImporter : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaImporter"; public override string ImporterType => "MorskaImporter";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaImporter> _logger;
public MorskaImporter( public MorskaImporter(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaImporter> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -52,8 +65,12 @@ public class MorskaImporter : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -66,25 +83,69 @@ public class MorskaImporter : MorskaBaseImporter
var newRecords = new List<Record>(); var newRecords = new List<Record>();
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); try
var data = dataRangeResponse.Values;
for (var i = 0; i < data[1].Count; i++)
{ {
if (!(data[0][i].ToString()?.Length > 0) || var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue; var data = dataRangeResponse.Values;
var record = new Record
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null && data.Count >= 2)
{ {
Id = Guid.NewGuid(), for (var i = 0; i < data[1].Count; i++)
Code = data[0][i].ToString(), {
Value1 = value, if (!(data[0][i].ToString()?.Length > 0) ||
CreatedAt = DateTime.UtcNow, !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
ModifiedAt = DateTime.UtcNow {
}; _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
newRecords.Add(record); continue;
}
var record = new Record
{
Id = Guid.NewGuid(),
Code = data[0][i].ToString(),
Value1 = value,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
newRecords.Add(record);
}
}
_db.Layers.Add(layer);
SaveRecords(layer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
} }
_db.Layers.Add(layer); catch (Exception e)
// TODO: Save records to the layer {
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
_db.SaveChanges(); throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -321,7 +321,9 @@ public class LayersController : Controller
.Where(x => .Where(x =>
x.Name != null && x.Name.Contains(nameFilter) && x.Name != null && x.Name.Contains(nameFilter) &&
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True"
&& x.Number == 7266
)
) )
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.AsNoTracking() .AsNoTracking()

View File

@@ -1,3 +1,3 @@
### ###
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/D1- GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-