imports: save records

This commit is contained in:
Michał Zieliński
2025-06-02 20:11:29 +02:00
parent 5046a29ee1
commit 8efd1edfad
6 changed files with 366 additions and 119 deletions

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaD1Importer> _logger;
public MorskaD1Importer( public MorskaD1Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaD1Importer> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -64,10 +79,15 @@ public class MorskaD1Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
try
{
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values; var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
var newRecords = (from t in data var newRecords = (from t in data
where t.Count > 1 && (string)t[0] != string.Empty where t.Count > 1 && (string)t[0] != string.Empty
select new Record select new Record
@@ -92,10 +112,41 @@ public class MorskaD1Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}).ToList(); }).ToList();
_db.Layers.Add(layer); _db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); SaveRecords(layer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
private double? ParseValue(string? value) private double? ParseValue(string? value)
@@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
return result; return result;
} }
catch (FormatException) catch (FormatException e)
{ {
_logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
return null; return null;
} }
} }
private bool IndexExists(IList<object> array, int index) private bool IndexExists(IList<object> array, int index)
{ {
return array != null && index >= 0 && index < array.Count; return array != null && index >= 0 && index < array.Count;

View File

@@ -1,6 +1,7 @@
using System.Globalization; using System.Globalization;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -9,52 +10,82 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaD3Importer : MorskaBaseImporter public class MorskaD3Importer : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaD3"; public override string ImporterType => "MorskaD3";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<MorskaD3Importer> _logger;
public MorskaD3Importer( public MorskaD3Importer(
AppDbContext db) AppDbContext db,
ILogger<MorskaD3Importer> logger)
{ {
_db = db; _db = db;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
throw new Exception($"ImportName not found, {importWorker.Name}"); throw new Exception($"ImportName not found, {importWorker.Name}");
} }
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
if (name == null) if (type == null)
{ {
throw new Exception($"ImportType not found, {importWorker.Name}"); throw new Exception($"ImportType not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
if (dataInbox == null) if (dataInbox == null)
{ {
throw new Exception($"DataInbox not found, {type}"); throw new Exception($"DataInbox not found, {type}");
} }
_logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
dataInbox.Id, dataInbox.CreatedAt);
try
{
var data = Convert.FromBase64String(dataInbox.Data); var data = Convert.FromBase64String(dataInbox.Data);
var tst = Encoding.UTF8.GetString(data); var jsonString = Encoding.UTF8.GetString(data);
var records = JsonSerializer.Deserialize<List<Record>>(Encoding.UTF8.GetString(data));
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
if (records == null) if (records == null)
{ {
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
} }
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
if (records.Count == 0) if (records.Count == 0)
{ {
throw new Exception($"No records found for {year}{month}"); throw new Exception($"No records found for {year}{month}");
} }
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
records.Count, year, month);
records = records.Select(x => records = records.Select(x =>
{ {
x.Id = Guid.NewGuid(); x.Id = Guid.NewGuid();
@@ -62,8 +93,10 @@ public class MorskaD3Importer : MorskaBaseImporter
x.ModifiedAt = DateTime.UtcNow; x.ModifiedAt = DateTime.UtcNow;
return x; return x;
}).ToList(); }).ToList();
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -72,11 +105,41 @@ public class MorskaD3Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
_db.Layers.Add(layer); _db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, records, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); SaveRecords(layer.Id, records);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
records.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaFk2Importer : MorskaBaseImporter public class MorskaFk2Importer : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaFK2"; public override string ImporterType => "MorskaFK2";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaFk2Importer> _logger;
public MorskaFk2Importer( public MorskaFk2Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaFk2Importer> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -52,8 +65,12 @@ public class MorskaFk2Importer : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -62,26 +79,45 @@ public class MorskaFk2Importer : MorskaBaseImporter
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
var newRecords = new List<Record>(); var newRecords = new List<Record>();
try
{
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values; var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null)
{
for (var i = 0; i < data.Count; i++) for (var i = 0; i < data.Count; i++)
{ {
if (data[i].Count <= 9 || (string)data[i][3] == string.Empty) continue; if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i);
continue;
}
var dateArr = data[i][1].ToString()!.Split("."); var dateArr = data[i][1].ToString()!.Split(".");
if (dateArr.Length != 3) if (dateArr.Length != 3)
{ {
_logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]);
throw new Exception($"Invalid date in row {i}"); throw new Exception($"Invalid date in row {i}");
} }
var number = data[i][2].ToString()!; var number = data[i][2].ToString()!;
if (number.Length == 1) number = $"0{number}"; if (number.Length == 1) number = $"0{number}";
var code = dateArr[2] + dateArr[1] + dateArr[0] + number; var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
if (!(data[i][9].ToString()?.Length > 0) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue; if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i);
continue;
}
var record = new Record var record = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -93,9 +129,41 @@ public class MorskaFk2Importer : MorskaBaseImporter
}; };
newRecords.Add(record); newRecords.Add(record);
} }
}
_db.Layers.Add(layer); _db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); SaveRecords(layer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -1,5 +1,6 @@
using System.Globalization; using System.Globalization;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models; using DiunaBI.Core.Models;
using DiunaBI.Database.Context; using DiunaBI.Database.Context;
@@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaImporter : MorskaBaseImporter public class MorskaImporter : MorskaBaseImporter
{ {
public override string ImporterType => "MorskaImporter"; public override string ImporterType => "MorskaImporter";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaImporter> _logger;
public MorskaImporter( public MorskaImporter(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues) SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaImporter> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger;
} }
public override void Import(Layer importWorker) public override void Import(Layer importWorker)
{ {
_logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})",
importWorker.Name, importWorker.Id);
var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
if (sheetId == null) if (sheetId == null)
{ {
throw new Exception($"SheetId not found, {importWorker.Name}"); throw new Exception($"SheetId not found, {importWorker.Name}");
} }
var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
if (sheetTabName == null) if (sheetTabName == null)
{ {
throw new Exception($"SheetTabName not found, {importWorker.Name}"); throw new Exception($"SheetTabName not found, {importWorker.Name}");
} }
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
if (year == null) if (year == null)
{ {
throw new Exception($"ImportYear not found, {importWorker.Name}"); throw new Exception($"ImportYear not found, {importWorker.Name}");
} }
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
if (month == null) if (month == null)
{ {
throw new Exception($"ImportMonth not found, {importWorker.Name}"); throw new Exception($"ImportMonth not found, {importWorker.Name}");
} }
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
if (name == null) if (name == null)
{ {
@@ -52,8 +65,12 @@ public class MorskaImporter : MorskaBaseImporter
throw new Exception($"DataRange not found, {importWorker.Name}"); throw new Exception($"DataRange not found, {importWorker.Name}");
} }
_logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
sheetId, sheetTabName, dataRange);
var layer = new Layer var layer = new Layer
{ {
Id = Guid.NewGuid(),
Number = _db.Layers.Count() + 1, Number = _db.Layers.Count() + 1,
ParentId = importWorker.Id, ParentId = importWorker.Id,
Type = LayerType.Import, Type = LayerType.Import,
@@ -66,12 +83,24 @@ public class MorskaImporter : MorskaBaseImporter
var newRecords = new List<Record>(); var newRecords = new List<Record>();
try
{
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values; var data = dataRangeResponse.Values;
_logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
if (data != null && data.Count >= 2)
{
for (var i = 0; i < data[1].Count; i++) for (var i = 0; i < data[1].Count; i++)
{ {
if (!(data[0][i].ToString()?.Length > 0) || if (!(data[0][i].ToString()?.Length > 0) ||
!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue; !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
{
_logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
continue;
}
var record = new Record var record = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -82,9 +111,41 @@ public class MorskaImporter : MorskaBaseImporter
}; };
newRecords.Add(record); newRecords.Add(record);
} }
}
_db.Layers.Add(layer); _db.Layers.Add(layer);
// TODO: Save records to the layer
//controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); SaveRecords(layer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
newRecords.Count, layer.Name, layer.Id);
}
catch (Exception e)
{
_logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
throw;
}
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
} }
} }

View File

@@ -321,7 +321,9 @@ public class LayersController : Controller
.Where(x => .Where(x =>
x.Name != null && x.Name.Contains(nameFilter) && x.Name != null && x.Name.Contains(nameFilter) &&
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True"
&& x.Number == 7266
)
) )
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.AsNoTracking() .AsNoTracking()

View File

@@ -1,3 +1,3 @@
### ###
GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/D1- GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-