From 8efd1edfade8f94b7aa10a230f2cc1ac43507d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Mon, 2 Jun 2025 20:11:29 +0200 Subject: [PATCH] imports: save records --- .../Importers/MorskaD1Importer.cs | 121 +++++++++++----- .../Importers/MorskaD3Importer.cs | 135 +++++++++++++----- .../Importers/MorskaFK2Importer.cs | 126 ++++++++++++---- .../Importers/MorskaImporter.cs | 97 ++++++++++--- .../Controllers/LayersController.cs | 4 +- tools/http-tests/AutoImport.http | 2 +- 6 files changed, 366 insertions(+), 119 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 23d987e..322cba4 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -1,5 +1,6 @@ using System.Globalization; using Google.Apis.Sheets.v4; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -11,37 +12,47 @@ public class MorskaD1Importer : MorskaBaseImporter private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; public MorskaD1Importer( AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } - public override void Import(Layer importWorker) + public override void Import(Layer importWorker) { + _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", + importWorker.Name, importWorker.Id); + var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } + var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } + var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; if (year == null) { throw new Exception($"ImportYear not found, {importWorker.Name}"); } + var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; if (month == null) { throw new Exception($"ImportMonth not found, {importWorker.Name}"); } + var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; if (name == null) { @@ -54,8 +65,12 @@ public class MorskaD1Importer : MorskaBaseImporter throw new Exception($"DataRange not found, {importWorker.Name}"); } + _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + sheetId, sheetTabName, dataRange); + var layer = new Layer { + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -64,38 +79,74 @@ public class MorskaD1Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; + layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - var newRecords = (from t in data - where t.Count > 1 && (string)t[0] != string.Empty - select new Record - { - Id = Guid.NewGuid(), - Code = t[0].ToString(), - Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null, - Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null, - Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null, - Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null, - Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null, - Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null, - Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null, - Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null, - Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null, - Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null, - Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null, - Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null, - Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null, - Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null, - Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }).ToList(); - _db.Layers.Add(layer); - // TODO: Save records to the layer - //controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - _db.SaveChanges(); + try + { + var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var data = dataRangeResponse.Values; + + _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + + var newRecords = (from t in data + where t.Count > 1 && (string)t[0] != string.Empty + select new Record + { + Id = Guid.NewGuid(), + Code = t[0].ToString(), + Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null, + Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null, + Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null, + Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null, + Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null, + Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null, + Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null, + Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null, + Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null, + Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null, + Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null, + Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null, + Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null, + Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null, + Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }).ToList(); + + _db.Layers.Add(layer); + + SaveRecords(layer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + newRecords.Count, layer.Name, layer.Id); + } + catch (Exception e) + { + _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId); + throw; + } + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } private double? ParseValue(string? value) @@ -107,11 +158,13 @@ public class MorskaD1Importer : MorskaBaseImporter double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); return result; } - catch (FormatException) + catch (FormatException e) { + _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message); return null; } } + private bool IndexExists(IList array, int index) { return array != null && index >= 0 && index < array.Count; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 19877bc..59f5afb 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text; using System.Text.Json; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -9,74 +10,136 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD3Importer : MorskaBaseImporter { public override string ImporterType => "MorskaD3"; + private readonly AppDbContext _db; + private readonly ILogger _logger; public MorskaD3Importer( - AppDbContext db) + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } + public override void Import(Layer importWorker) { + _logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})", + importWorker.Name, importWorker.Id); + var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; if (year == null) { throw new Exception($"ImportYear not found, {importWorker.Name}"); } + var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; if (month == null) { throw new Exception($"ImportMonth not found, {importWorker.Name}"); } + var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; if (name == null) { throw new Exception($"ImportName not found, {importWorker.Name}"); } + var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; - if (name == null) + if (type == null) { throw new Exception($"ImportType not found, {importWorker.Name}"); } + + _logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type); + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); if (dataInbox == null) { throw new Exception($"DataInbox not found, {type}"); } - var data = Convert.FromBase64String(dataInbox.Data); - var tst = Encoding.UTF8.GetString(data); - var records = JsonSerializer.Deserialize>(Encoding.UTF8.GetString(data)); - if (records == null) - { - throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); - } - records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); - if (records.Count == 0) - { - throw new Exception($"No records found for {year}{month}"); - } - records = records.Select(x => - { - x.Id = Guid.NewGuid(); - x.CreatedAt = DateTime.UtcNow; - x.ModifiedAt = DateTime.UtcNow; - return x; - }).ToList(); - var layer = new Layer - { - Number = _db.Layers.Count() + 1, - ParentId = importWorker.Id, - Type = LayerType.Import, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; - _db.Layers.Add(layer); - // TODO: Save records to the layer - //controller.SaveRecords(layer.Id, records, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - _db.SaveChanges(); + _logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}", + dataInbox.Id, dataInbox.CreatedAt); + + try + { + var data = Convert.FromBase64String(dataInbox.Data); + var jsonString = Encoding.UTF8.GetString(data); + + _logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length); + + var records = JsonSerializer.Deserialize>(jsonString); + if (records == null) + { + throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); + } + + _logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count); + + records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); + if (records.Count == 0) + { + throw new Exception($"No records found for {year}{month}"); + } + + _logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}", + records.Count, year, month); + + records = records.Select(x => + { + x.Id = Guid.NewGuid(); + x.CreatedAt = DateTime.UtcNow; + x.ModifiedAt = DateTime.UtcNow; + return x; + }).ToList(); + + var layer = new Layer + { + Id = Guid.NewGuid(), + Number = _db.Layers.Count() + 1, + ParentId = importWorker.Id, + Type = LayerType.Import, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + + _db.Layers.Add(layer); + + SaveRecords(layer.Id, records); + _db.SaveChanges(); + + _logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + records.Count, layer.Name, layer.Id); + } + catch (Exception e) + { + _logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id); + throw; + } + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 9bac27d..62c9de0 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -1,5 +1,6 @@ using System.Globalization; using Google.Apis.Sheets.v4; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaFk2Importer : MorskaBaseImporter { public override string ImporterType => "MorskaFK2"; + private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public MorskaFk2Importer( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } public override void Import(Layer importWorker) { + _logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})", + importWorker.Name, importWorker.Id); + var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } + var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } + var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; if (year == null) { throw new Exception($"ImportYear not found, {importWorker.Name}"); } + var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; if (month == null) { throw new Exception($"ImportMonth not found, {importWorker.Name}"); } + var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; if (name == null) { @@ -52,8 +65,12 @@ public class MorskaFk2Importer : MorskaBaseImporter throw new Exception($"DataRange not found, {importWorker.Name}"); } + _logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + sheetId, sheetTabName, dataRange); + var layer = new Layer { + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -62,40 +79,91 @@ public class MorskaFk2Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture)}"; + layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - for (var i = 0; i < data.Count; i++) + try { - if (data[i].Count <= 9 || (string)data[i][3] == string.Empty) continue; - var dateArr = data[i][1].ToString()!.Split("."); - if (dateArr.Length != 3) + var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var data = dataRangeResponse.Values; + + _logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + + if (data != null) { - throw new Exception($"Invalid date in row {i}"); + for (var i = 0; i < data.Count; i++) + { + if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) + { + _logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i); + continue; + } + + var dateArr = data[i][1].ToString()!.Split("."); + if (dateArr.Length != 3) + { + _logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]); + throw new Exception($"Invalid date in row {i}"); + } + + var number = data[i][2].ToString()!; + if (number.Length == 1) number = $"0{number}"; + var code = dateArr[2] + dateArr[1] + dateArr[0] + number; + + if (string.IsNullOrEmpty(data[i][9]?.ToString()) || + !double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i); + continue; + } + + var record = new Record + { + Id = Guid.NewGuid(), + Code = code, + Desc1 = data[i][3].ToString(), + Value1 = value, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + newRecords.Add(record); + } } - var number = data[i][2].ToString()!; - if (number.Length == 1) number = $"0{number}"; - var code = dateArr[2] + dateArr[1] + dateArr[0] + number; - if (!(data[i][9].ToString()?.Length > 0) || - !double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue; - var record = new Record - { - Id = Guid.NewGuid(), - Code = code, - Desc1 = data[i][3].ToString(), - Value1 = value, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - newRecords.Add(record); + _db.Layers.Add(layer); + + SaveRecords(layer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + newRecords.Count, layer.Name, layer.Id); } - _db.Layers.Add(layer); - // TODO: Save records to the layer - //controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - _db.SaveChanges(); + catch (Exception e) + { + _logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId); + throw; + } + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs index 73c4de7..b655f15 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs @@ -1,5 +1,6 @@ using System.Globalization; using Google.Apis.Sheets.v4; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -8,38 +9,50 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaImporter : MorskaBaseImporter { public override string ImporterType => "MorskaImporter"; + private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public MorskaImporter( AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } public override void Import(Layer importWorker) { + _logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})", + importWorker.Name, importWorker.Id); + var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; if (sheetId == null) { throw new Exception($"SheetId not found, {importWorker.Name}"); } + var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; if (sheetTabName == null) { throw new Exception($"SheetTabName not found, {importWorker.Name}"); } + var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; if (year == null) { throw new Exception($"ImportYear not found, {importWorker.Name}"); } + var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; if (month == null) { throw new Exception($"ImportMonth not found, {importWorker.Name}"); } + var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; if (name == null) { @@ -52,8 +65,12 @@ public class MorskaImporter : MorskaBaseImporter throw new Exception($"DataRange not found, {importWorker.Name}"); } + _logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + sheetId, sheetTabName, dataRange); + var layer = new Layer { + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -66,25 +83,69 @@ public class MorskaImporter : MorskaBaseImporter var newRecords = new List(); - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - for (var i = 0; i < data[1].Count; i++) + try { - if (!(data[0][i].ToString()?.Length > 0) || - !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) continue; - var record = new Record + var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var data = dataRangeResponse.Values; + + _logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + + if (data != null && data.Count >= 2) { - Id = Guid.NewGuid(), - Code = data[0][i].ToString(), - Value1 = value, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - newRecords.Add(record); + for (var i = 0; i < data[1].Count; i++) + { + if (!(data[0][i].ToString()?.Length > 0) || + !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i); + continue; + } + + var record = new Record + { + Id = Guid.NewGuid(), + Code = data[0][i].ToString(), + Value1 = value, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + newRecords.Add(record); + } + } + + _db.Layers.Add(layer); + + SaveRecords(layer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + newRecords.Count, layer.Name, layer.Id); } - _db.Layers.Add(layer); - // TODO: Save records to the layer - //controller.SaveRecords(layer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); - _db.SaveChanges(); + catch (Exception e) + { + _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId); + throw; + } + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 8f67a2f..400aa90 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -321,7 +321,9 @@ public class LayersController : Controller .Where(x => x.Name != null && x.Name.Contains(nameFilter) && x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True" + && x.Number == 7266 + ) ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index d8cfd30..784b5ed 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/D1- +GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-