From c7d9acead044e8e665e714d034def5340203be20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 5 Dec 2025 09:51:04 +0100 Subject: [PATCH] UI refactor (structure cleanup) --- .claude/commands/context.md | 3 + .claude/commands/updateContext.md | 18 + .claude/project-context.md | 669 ++++++++++++++++++ .../Components/{ => Auth}/AuthGuard.razor | 0 .../Components/{ => Auth}/LoginCard.razor | 0 .../{ => Components/Layout}/EmptyLayout.razor | 0 .../{ => Components/Layout}/MainLayout.razor | 0 .../Components/{ => Layout}/Routes.razor | 0 .../{Components => Pages}/Dashboard.razor | 0 .../Details.razor} | 2 - .../Details.razor.cs} | 11 +- .../DataInbox/Index.razor} | 7 + .../DataInbox/Index.razor.cs} | 4 +- .../Pages/DataInboxListPage.razor | 8 - .../{Components => Pages}/Index.razor | 0 DiunaBI.UI.Shared/Pages/JobListPage.razor | 8 - .../Details.razor} | 0 .../Jobs/Index.razor} | 7 + .../Jobs/Index.razor.cs} | 4 +- DiunaBI.UI.Shared/Pages/LayerListPage.razor | 8 - .../Details.razor} | 4 - .../Details.razor.cs} | 17 +- .../Layers/Index.razor} | 10 +- .../Layers/Index.razor.cs} | 8 +- .../Pages/{LoginPage.razor => Login.razor} | 0 DiunaBI.UI.Shared/_Imports.razor | 2 + 26 files changed, 746 insertions(+), 44 deletions(-) create mode 100644 .claude/commands/context.md create mode 100644 .claude/commands/updateContext.md create mode 100644 .claude/project-context.md rename DiunaBI.UI.Shared/Components/{ => Auth}/AuthGuard.razor (100%) rename DiunaBI.UI.Shared/Components/{ => Auth}/LoginCard.razor (100%) rename DiunaBI.UI.Shared/{ => Components/Layout}/EmptyLayout.razor (100%) rename DiunaBI.UI.Shared/{ => Components/Layout}/MainLayout.razor (100%) rename DiunaBI.UI.Shared/Components/{ => Layout}/Routes.razor (100%) rename DiunaBI.UI.Shared/{Components => Pages}/Dashboard.razor (100%) rename DiunaBI.UI.Shared/Pages/{DataInboxDetailPage.razor => DataInbox/Details.razor} (97%) rename DiunaBI.UI.Shared/Pages/{DataInboxDetailPage.razor.cs => DataInbox/Details.razor.cs} (86%) rename DiunaBI.UI.Shared/{Components/DataInboxListComponent.razor => Pages/DataInbox/Index.razor} (95%) rename DiunaBI.UI.Shared/{Components/DataInboxListComponent.razor.cs => Pages/DataInbox/Index.razor.cs} (95%) delete mode 100644 DiunaBI.UI.Shared/Pages/DataInboxListPage.razor rename DiunaBI.UI.Shared/{Components => Pages}/Index.razor (100%) delete mode 100644 DiunaBI.UI.Shared/Pages/JobListPage.razor rename DiunaBI.UI.Shared/Pages/{JobDetailPage.razor => Jobs/Details.razor} (100%) rename DiunaBI.UI.Shared/{Components/JobListComponent.razor => Pages/Jobs/Index.razor} (97%) rename DiunaBI.UI.Shared/{Components/JobListComponent.razor.cs => Pages/Jobs/Index.razor.cs} (97%) delete mode 100644 DiunaBI.UI.Shared/Pages/LayerListPage.razor rename DiunaBI.UI.Shared/Pages/{LayerDetailPage.razor => Layers/Details.razor} (99%) rename DiunaBI.UI.Shared/Pages/{LayerDetailPage.razor.cs => Layers/Details.razor.cs} (97%) rename DiunaBI.UI.Shared/{Components/LayerListComponent.razor => Pages/Layers/Index.razor} (97%) rename DiunaBI.UI.Shared/{Components/LayerListComponent.razor.cs => Pages/Layers/Index.razor.cs} (95%) rename DiunaBI.UI.Shared/Pages/{LoginPage.razor => Login.razor} (100%) diff --git a/.claude/commands/context.md b/.claude/commands/context.md new file mode 100644 index 0000000..439fbcb --- /dev/null +++ b/.claude/commands/context.md @@ -0,0 +1,3 @@ +Read the project context file at `.claude/project-context.md` to quickly understand the DiunaBI project structure, architecture, key components, and recent development focus. This will bootstrap your knowledge without needing to explore the entire codebase. + +After reading the context file, briefly acknowledge what you've learned and ask the user what they need help with. diff --git a/.claude/commands/updateContext.md b/.claude/commands/updateContext.md new file mode 100644 index 0000000..90ced5e --- /dev/null +++ b/.claude/commands/updateContext.md @@ -0,0 +1,18 @@ +Re-explore the DiunaBI codebase comprehensively and update the `.claude/project-context.md` file with the latest: + +1. Project structure changes (new projects, folders, or major reorganization) +2. New features and functionality added +3. Database schema changes (new entities, significant field additions) +4. New API endpoints or controllers +5. New services or business logic components +6. Changes to authentication/authorization +7. New external integrations or plugins +8. Recent git commits (last 5-10) to understand development direction +9. Updated configuration or environment changes +10. New design patterns or architectural changes + +Use the Task tool with subagent_type='Explore' and thoroughness level 'very thorough' to comprehensively analyze the codebase. + +After exploration, update the `.claude/project-context.md` file with all relevant changes while maintaining the existing structure and format. Preserve information that hasn't changed. + +When done, provide a brief summary of what was updated in the context file. diff --git a/.claude/project-context.md b/.claude/project-context.md new file mode 100644 index 0000000..c760642 --- /dev/null +++ b/.claude/project-context.md @@ -0,0 +1,669 @@ +# DiunaBI Project Context + +> This file is auto-generated for Claude Code to quickly understand the project structure. +> Last updated: 2025-12-05 + +## RECENT CHANGES (This Session) + +**UI Reorganization (Dec 5, 2025):** +- ✅ Moved pages to feature-based folders: `Pages/Layers/`, `Pages/Jobs/`, `Pages/DataInbox/` +- ✅ Organized components: `Components/Layout/` (MainLayout, EmptyLayout, Routes), `Components/Auth/` (AuthGuard, LoginCard) +- ✅ Removed obsolete wrapper files (LayerListPage, JobListPage, DataInboxListPage, etc.) +- ✅ Removed duplicate component files (LayerListComponent, JobListComponent, DataInboxListComponent) +- ✅ Standardized code-behind: `.razor.cs` for complex logic, inline `@code` for simple pages +- ✅ Updated `_Imports.razor` with new namespaces: `DiunaBI.UI.Shared.Components.Layout`, `DiunaBI.UI.Shared.Components.Auth` +- ✅ All routes unchanged - backward compatible + +--- + +## PROJECT TYPE & TECH STACK + +**Application Type:** Full-stack Business Intelligence (BI) platform with multi-tier architecture, real-time capabilities, and plugin system + +**Core Stack:** +- Backend: ASP.NET Core 10.0 Web API +- Frontend: Blazor Server + MAUI Mobile +- Database: SQL Server + EF Core 10.0 +- UI: MudBlazor 8.0 +- Real-time: SignalR (EntityChangeHub) +- Google: Sheets API, Drive API, OAuth +- Logging: Serilog (Console, File, Seq) +- Auth: JWT Bearer + Google OAuth + +--- + +## SOLUTION STRUCTURE (10 Projects) + +``` +DiunaBI.API (Web API) +├── Controllers: Auth, Layers, Jobs, DataInbox +├── Hubs: EntityChangeHub (SignalR real-time updates) +└── Services: GoogleAuth, JwtToken + +DiunaBI.Domain (Entities) +└── User, Layer, Record, RecordHistory, QueueJob, DataInbox, ProcessSource + +DiunaBI.Application (DTOs) +└── LayerDto, RecordDto, UserDto, RecordHistoryDto, PagedResult, JobDto + +DiunaBI.Infrastructure (Data + Services) +├── Data: AppDbContext, Migrations (47 total) +├── Interceptors: EntityChangeInterceptor (auto-broadcasts DB changes) +├── Services: PluginManager, JobScheduler, JobWorker, GoogleSheets/Drive +├── Plugins: BaseDataImporter, BaseDataProcessor, BaseDataExporter +└── Interfaces: IPlugin, IDataProcessor, IDataImporter, IDataExporter + +DiunaBI.UI.Web (Blazor Server) +└── Server-side Blazor web application + +DiunaBI.UI.Mobile (MAUI) +└── iOS, Android, Windows, macOS support + +DiunaBI.UI.Shared (Blazor Component Library - Reorganized) +├── Pages/ +│ ├── Layers/ (Index.razor, Details.razor) +│ ├── Jobs/ (Index.razor, Details.razor) +│ ├── DataInbox/ (Index.razor, Details.razor) +│ ├── Dashboard.razor, Login.razor, Index.razor +├── Components/ +│ ├── Layout/ (MainLayout, EmptyLayout, Routes) +│ └── Auth/ (AuthGuard, LoginCard) +└── Services/ + ├── LayerService, JobService, DataInboxService + ├── EntityChangeHubService (SignalR client) + ├── FilterStateServices (remember filters) + └── AuthService, TokenProvider + +DiunaBI.Plugins.Morska (Feature Plugin) +├── Importers: Standard, D1, D3, FK2 (4 total) +├── Processors: D6, T1, T3, T4, T5 variants (12 total) +└── Exporters: Google Sheets export (1) + +DiunaBI.Plugins.PedrolloPL (Feature Plugin - NEW) +└── Importers: P2 (1 total) + +DiunaBI.Tests (Testing) +└── Unit and integration tests +``` + +--- + +## CORE FUNCTIONALITY + +**Purpose:** BI platform for data import, processing, transformation via modular plugin architecture. Multi-layer workflows with audit trails, real-time notifications, scheduled job processing. + +**Main Features:** +1. **Layer Management** - 4 types (Import/Processed/Admin/Dictionary), parent-child relationships, soft deletes +2. **Data Records** - 32 numeric columns (Value1-32) + description, hierarchical, full audit trail +3. **Plugin Architecture** - Dynamic assembly loading, base classes in Infrastructure, 3 types (Importers/Processors/Exporters) +4. **Job Queue System** - Background worker with retry logic (30s → 2m → 5m), priority-based, auto-scheduling +5. **External Data** - DataInbox API, Google Sheets read/write, Google Drive integration +6. **Real-time Updates** - SignalR broadcasts entity changes (create/update/delete) to all connected clients +7. **Audit Trail** - RecordHistory tracks all record changes with field-level diffs and JSON summaries +8. **Filter Persistence** - UI filter states saved across sessions (LayerFilterStateService, DataInboxFilterStateService) + +--- + +## KEY ENTITIES + +**Layer** +- Id, Number, Name, Type (Import/Processed/Administration/Dictionary) +- CreatedAt/ModifiedAt, CreatedBy/ModifiedBy (with user relations) +- IsDeleted (soft delete), IsCancelled (processing control), ParentId +- Relations: Records (1-to-many), ProcessSources (1-to-many) + +**Record** +- Id, Code (unique identifier), LayerId +- Value1-Value32 (double?), Desc1 (string, max 10000 chars) +- CreatedAt/ModifiedAt, CreatedBy/ModifiedBy, IsDeleted +- Audit: Full history tracked in RecordHistory table + +**RecordHistory** (NEW - Migration 47) +- RecordId, LayerId, ChangedAt, ChangedById +- ChangeType (Created/Updated/Deleted) +- Code, Desc1 (snapshot at time of change) +- ChangedFields (comma-separated field names) +- ChangesSummary (JSON with old/new values) +- Indexes: (RecordId, ChangedAt), (LayerId, ChangedAt) for performance + +**QueueJob** +- LayerId, LayerName, PluginName +- JobType (Import/Process) +- Priority (0 = highest), Status (Pending/Running/Completed/Failed/Retrying) +- RetryCount, MaxRetries (default 5) +- CreatedAt, LastAttemptAt, CompletedAt +- LastError (detailed error message) + +**DataInbox** +- Id, Name, Source (identifiers) +- Data (base64-encoded JSON array) +- CreatedAt +- Used by importers to stage incoming data + +**User** +- Id (Guid), Email, UserName +- CreatedAt, LastLoginAt +- Google OAuth identity + +**ProcessSource** +- Id, SourceLayerId, TargetLayerId +- Defines layer processing relationships + +--- + +## API ENDPOINTS + +**Base:** `/` (ApiController routes) + +### AuthController (/auth) +- `POST /auth/apiToken` - Exchange Google ID token for JWT (AllowAnonymous) +- `POST /auth/refresh` - Refresh expired JWT token + +### LayersController (/layers) +- `GET /layers?page=1&pageSize=10&search=&type=` - List layers (paged, filterable) +- `GET /layers/{id}` - Get layer details with records +- `POST /layers` - Create new layer +- `PUT /layers/{id}` - Update layer +- `DELETE /layers/{id}` - Soft delete layer +- `POST /layers/{id}/records` - Add/update records +- `PUT /layers/{layerId}/records/{recordId}` - Update specific record +- `DELETE /layers/{layerId}/records/{recordId}` - Delete record +- `GET /layers/{layerId}/records/{recordId}/history` - Get record history +- `GET /layers/{layerId}/deleted-records` - Get deleted records with history + +### JobsController (/jobs) - NEW +- `GET /jobs?page=1&pageSize=50&status=&jobType=` - List jobs (paged, filterable) +- `GET /jobs/{id}` - Get job details +- `GET /jobs/stats` - Get job statistics (counts by status) +- `POST /jobs/schedule/{apiKey}` - Schedule all jobs from layer configs +- `POST /jobs/schedule/imports/{apiKey}` - Schedule import jobs only +- `POST /jobs/schedule/processes/{apiKey}` - Schedule process jobs only +- `POST /jobs/create-for-layer/{layerId}` - Create job for specific layer (manual trigger) +- `POST /jobs/{id}/retry` - Retry failed job (resets to Pending) +- `DELETE /jobs/{id}` - Cancel pending/retrying job + +### DataInboxController (/datainbox) +- `GET /datainbox?page=1&pageSize=10&search=` - List inbox items (paged, filterable) +- `GET /datainbox/{id}` - Get inbox item with decoded data +- `POST /datainbox` - Create inbox item +- `PUT /datainbox/Add/{apiKey}` - Add data (API key + Basic Auth) +- `DELETE /datainbox/{id}` - Delete inbox item + +### SignalR Hub +- `/hubs/entitychanges` - SignalR hub for real-time entity change notifications + - Event: `EntityChanged(module, id, operation)` - broadcasts to all clients + - Modules: QueueJobs, Layers, Records, RecordHistory + +--- + +## AUTHENTICATION & SECURITY + +**Flow:** +1. Client exchanges Google ID token → `/auth/apiToken` +2. GoogleAuthService validates token with Google, maps to internal User +3. Returns JWT (7-day expiration, HS256 signing) +4. JWT required on all protected endpoints (except /auth/apiToken, /health) +5. UserId extraction middleware sets X-UserId header for audit trails + +**Security:** +- Google OAuth 2.0 for identity verification +- JWT Bearer tokens for API access +- API key + Basic Auth for DataInbox external endpoints +- CORS configured for: + - http://localhost:4200 + - https://diuna.bim-it.pl + - https://morska.diunabi.com + +--- + +## KEY SERVICES + +### Infrastructure Services + +**PluginManager** +- Location: `DiunaBI.Infrastructure/Services/PluginManager.cs` +- Loads plugin assemblies from `bin/Plugins/` directory at startup +- Registers IDataProcessor, IDataImporter, IDataExporter implementations +- Provides plugin discovery and execution + +**JobSchedulerService** +- Location: `DiunaBI.Infrastructure/Services/JobSchedulerService.cs` +- Creates QueueJob entries from Administration layer configs +- Reads layer.Records with Code="Plugin", Code="Priority", Code="MaxRetries" +- Methods: ScheduleImportJobsAsync, ScheduleProcessJobsAsync, ScheduleAllJobsAsync + +**JobWorkerService** (BackgroundService) +- Location: `DiunaBI.Infrastructure/Services/JobWorkerService.cs` +- Polls QueueJobs table every 10 seconds +- Executes jobs via PluginManager (Import/Process) +- Retry logic with exponential backoff: 30s → 2m → 5m delays +- Rate limiting: 5-second delay after imports (Google Sheets API quota) +- Updates job status in real-time (triggers SignalR broadcasts) + +**EntityChangeInterceptor** +- Location: `DiunaBI.Infrastructure/Interceptors/EntityChangeInterceptor.cs` +- EF Core SaveChangesInterceptor +- Captures entity changes: Added, Modified, Deleted +- Broadcasts changes via SignalR EntityChangeHub after successful save +- Uses reflection to avoid circular dependencies with IHubContext + +**GoogleSheetsHelper** +- Location: `DiunaBI.Infrastructure/Helpers/GoogleSheetsHelper.cs` +- Google Sheets API v4 integration +- Methods: ReadRange, WriteRange, CreateSpreadsheet, UpdateSpreadsheet + +**GoogleDriveHelper** +- Location: `DiunaBI.Infrastructure/Helpers/GoogleDriveHelper.cs` +- Google Drive API v3 integration +- Methods: UploadFile, ListFiles, MoveFile + +**GoogleAuthService / JwtTokenService** +- Authentication and token management +- JWT generation and validation + +### UI Services + +**EntityChangeHubService** +- Location: `DiunaBI.UI.Shared/Services/EntityChangeHubService.cs` +- Singleton service for SignalR client connection +- Auto-reconnect enabled +- Event: `EntityChanged` - UI components subscribe for real-time updates +- Initialized in MainLayout.OnInitializedAsync + +**LayerService / JobService / DataInboxService** +- HTTP clients for API communication +- DTOs serialization/deserialization +- Paged result handling + +**LayerFilterStateService / DataInboxFilterStateService** +- Persist filter state across navigation +- Singleton services remember search, type, page selections + +--- + +## DATABASE SCHEMA + +**Total Migrations:** 47 + +**Latest Migrations:** + +**Migration 47: RecordHistory (Dec 1, 2025)** +- **NEW Table: RecordHistory** +- Tracks all record changes (Created, Updated, Deleted) +- Fields: Id, RecordId, LayerId, ChangedAt, ChangedById, ChangeType, Code, Desc1, ChangedFields, ChangesSummary +- Indexes: IX_RecordHistory_RecordId_ChangedAt, IX_RecordHistory_LayerId_ChangedAt +- Foreign key: RecordHistory.ChangedById → Users.Id + +**Migration 46: FixLayerDefaultValues (Nov 20, 2025)** +- Set default value: Layers.IsDeleted = false + +**Migration 45: UpdateModel (Nov 19, 2025)** +- Added GETUTCDATE() defaults for all timestamp fields +- Changed foreign key constraints from CASCADE to RESTRICT: + - Layers → Users (CreatedById, ModifiedById) + - Records → Users (CreatedById, ModifiedById) +- Added FK_ProcessSources_Layers_LayerId + +**Core Tables:** +- Users (authentication, audit) +- Layers (4 types, soft deletes, parent-child) +- Records (32 Value fields + Desc1, audit, soft deletes) +- RecordHistory (change tracking, field diffs, JSON summaries) +- QueueJobs (job queue, retry logic, status tracking) +- DataInbox (incoming data staging, base64 encoded) +- ProcessSources (layer relationships) + +--- + +## PLUGIN SYSTEM + +### Base Classes (Infrastructure/Plugins/) + +**BaseDataImporter** (`DiunaBI.Infrastructure/Plugins/BaseDataImporter.cs`) +- Abstract base for all importers +- Methods: ImportAsync(layerId, jobId), ValidateConfiguration() +- Access: AppDbContext, PluginManager, GoogleSheetsHelper, GoogleDriveHelper + +**BaseDataProcessor** (`DiunaBI.Infrastructure/Plugins/BaseDataProcessor.cs`) +- Abstract base for all processors +- Methods: ProcessAsync(layerId, jobId), ValidateConfiguration() +- Access: AppDbContext, PluginManager + +**BaseDataExporter** (`DiunaBI.Infrastructure/Plugins/BaseDataExporter.cs`) +- Abstract base for all exporters +- Methods: ExportAsync(layerId, jobId), ValidateConfiguration() +- Access: AppDbContext, GoogleSheetsHelper, GoogleDriveHelper + +### Morska Plugin (DiunaBI.Plugins.Morska) + +**Importers (4):** +- MorskaStandardImporter - Generic CSV/Excel import +- MorskaD1Importer - D1 data format +- MorskaD3Importer - D3 data format +- MorskaFK2Importer - FK2 data format + +**Processors (12):** +- MorskaD6Processor +- MorskaT1R1Processor +- MorskaT1R3Processor +- MorskaT3SingleSourceProcessor +- MorskaT3SourceYearSummaryProcessor +- MorskaT3MultiSourceSummaryProcessor +- MorskaT3MultiSourceYearSummaryProcessor +- MorskaT4R2Processor +- MorskaT4SingleSourceProcessor +- MorskaT5LastValuesProcessor +- MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE (deprecated) +- MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE (deprecated) + +**Exporters (1):** +- googleSheet.export.cs - Google Sheets export + +**Total:** ~6,566 lines of code + +### PedrolloPL Plugin (DiunaBI.Plugins.PedrolloPL) - NEW + +**Importers (1):** +- **PedrolloPLImportP2** (`DiunaBI.Plugins.PedrolloPL/Importers/PedrolloPLImportP2.cs`) + - Imports P2 data from DataInbox + - Uses L1-D-P2-CODES dictionary layer for region code mapping + - Creates 12 monthly records per region (Value1-Value12) + - Generates Import layers: L{Number}-I-P2-{Year}-{Timestamp} + - Handles base64 JSON data decoding + +--- + +## UI STRUCTURE (DiunaBI.UI.Shared) + +### Reorganized Structure (Dec 5, 2025) + +**Pages/** (Routable pages with @page directive) +``` +Pages/ +├── Layers/ +│ ├── Index.razor + Index.razor.cs - /layers (list with filters, pagination) +│ └── Details.razor + Details.razor.cs - /layers/{id} (detail, edit, history) +├── Jobs/ +│ ├── Index.razor + Index.razor.cs - /jobs (list with filters, real-time updates) +│ └── Details.razor - /jobs/{id} (detail, retry, cancel, real-time) +├── DataInbox/ +│ ├── Index.razor + Index.razor.cs - /datainbox (list with filters) +│ └── Details.razor + Details.razor.cs - /datainbox/{id} (detail, base64 decode) +├── Dashboard.razor - /dashboard (user info) +├── Login.razor - /login (Google OAuth) +└── Index.razor - / (redirects to /dashboard) +``` + +**Components/** (Reusable components, no routes) +``` +Components/ +├── Layout/ +│ ├── MainLayout.razor - Main app layout with drawer, nav menu +│ ├── EmptyLayout.razor - Minimal layout for login page +│ └── Routes.razor - Router configuration +└── Auth/ + ├── AuthGuard.razor - Authentication guard wrapper + └── LoginCard.razor - Google login button component +``` + +**Navigation Menu:** +- Dashboard (/dashboard) - User profile +- Layers (/layers) - Layer management +- Data Inbox (/datainbox) - Incoming data review +- Jobs (/jobs) - Job queue monitoring (with real-time status updates) + +**Code-Behind Pattern:** +- Complex pages (50+ lines logic): Separate `.razor.cs` files +- Simple pages: Inline `@code` blocks +- Namespaces: `DiunaBI.UI.Shared.Pages.{Feature}` + +--- + +## REAL-TIME FEATURES (SignalR) + +### Architecture + +**Hub:** `DiunaBI.API/Hubs/EntityChangeHub.cs` +- Endpoint: `/hubs/entitychanges` +- Method: `SendEntityChange(string module, string id, string operation)` +- Broadcasts: `EntityChanged` event to all connected clients + +**Interceptor:** `DiunaBI.Infrastructure/Interceptors/EntityChangeInterceptor.cs` +- EF Core SaveChangesInterceptor +- Detects: Added, Modified, Deleted entities +- Broadcasts: After successful SaveChanges +- Modules: QueueJobs, Layers, Records, RecordHistory + +**UI Service:** `DiunaBI.UI.Shared/Services/EntityChangeHubService.cs` +- Singleton initialized in MainLayout +- Auto-reconnect enabled +- Components subscribe: `HubService.EntityChanged += OnEntityChanged` + +### Real-time Update Flow + +1. User action → API endpoint +2. DbContext.SaveChangesAsync() +3. EntityChangeInterceptor captures changes +4. SignalR broadcast to all clients: `EntityChanged(module, id, operation)` +5. UI components receive event and refresh data +6. StateHasChanged() updates UI + +**Example:** Job status changes appear instantly on JobDetailPage and JobListPage + +--- + +## JOB QUEUE SYSTEM + +### Components + +**Entity:** `QueueJob` (DiunaBI.Domain/Entities/QueueJob.cs) +- JobType: Import, Process +- JobStatus: Pending, Running, Completed, Failed, Retrying +- Priority: 0 = highest priority +- Retry: 30s → 2m → 5m delays, max 5 attempts + +**Scheduler:** `JobSchedulerService` +- Reads Administration layer configs (Type=ImportWorker/ProcessWorker) +- Auto-creates jobs based on layer.Records configuration +- API endpoints: `/jobs/schedule/{apiKey}`, `/jobs/schedule/imports/{apiKey}`, `/jobs/schedule/processes/{apiKey}` + +**Worker:** `JobWorkerService` (BackgroundService) +- Polls every 10 seconds +- Executes via PluginManager +- Exponential backoff on failures +- Rate limiting for Google API quota +- Real-time status updates via SignalR + +**UI:** `Pages/Jobs/` +- Index.razor - Job list with filters, real-time updates +- Details.razor - Job detail with retry/cancel, real-time status + +### Job Lifecycle + +1. **Creation** - JobSchedulerService or manual via API +2. **Queued** - Status: Pending, sorted by Priority +3. **Execution** - JobWorkerService picks up, Status: Running +4. **Completion** - Status: Completed or Failed +5. **Retry** - On failure, Status: Retrying with exponential backoff +6. **Real-time** - All status changes broadcast via SignalR + +**Statistics Endpoint:** `GET /jobs/stats` +```json +{ + "pending": 5, + "running": 2, + "completed": 150, + "failed": 3, + "retrying": 1, + "total": 161 +} +``` + +--- + +## RECENT DEVELOPMENT + +**Recent Commits (Dec 2-5, 2025):** +- **193127b:** SignalR for realtime entitychanges (Dec 4) +- **bf2beda, 942da18:** Build fixes (Dec 4) +- **a3fa8f9:** P2 import is working (Dec 4) +- **0e3b393:** WIP: p2 plugin (Dec 3) +- **445c07a:** Morska plugins refactor (Dec 2) +- **3f8e62f:** WIP: queue engine (Dec 2) +- **248106a:** Plugins little refactor (Dec 2) +- **587d4d6:** Pedrollo plugins (Dec 2) +- **e70a8dd:** Remember list filters (Dec 2) +- **89859cd:** Record history is working (Dec 1) + +**Development Focus (Last 30 Days):** +1. ✅ Real-time updates (SignalR integration) +2. ✅ Job queue system (background worker, retry logic) +3. ✅ PedrolloPL plugin (P2 importer) +4. ✅ Record history tracking (audit trail) +5. ✅ UI reorganization (feature-based folders) +6. ✅ Plugin refactoring (base classes in Infrastructure) +7. ✅ Filter persistence (UI state management) + +**Major Features Added:** +- SignalR real-time entity change notifications +- Background job processing with retry logic +- Record history with field-level diffs +- PedrolloPL P2 data importer +- UI reorganization (Pages/Layers, Pages/Jobs, Pages/DataInbox) +- Filter state persistence across sessions + +--- + +## CONFIGURATION + +**Key Settings (appsettings.Development.json):** +- ConnectionStrings:SQLDatabase - SQL Server (localhost:21433, DB: DiunaBI-PedrolloPL) +- JwtSettings:SecurityKey, ExpiryDays (7) +- GoogleAuth:ClientId, RedirectUri +- apiKey, apiUser, apiPass - DataInbox API security +- exportDirectory - Google Drive folder ID for exports +- apiLocalUrl - localhost:5400 +- InstanceName - DEV/PROD environment identifier + +**Logging Configuration:** +```json +"Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning", + "Microsoft.EntityFrameworkCore.Infrastructure": "Warning", + "System.Net.Http.HttpClient": "Warning", + "Google.Apis": "Warning", + "DiunaBI.Core.Services.PluginManager": "Information" + } + } +} +``` + +**CORS Origins:** +- http://localhost:4200 (development) +- https://diuna.bim-it.pl (production) +- https://morska.diunabi.com (production) + +--- + +## PATTERNS & ARCHITECTURE + +**Design Patterns:** +- Clean Architecture (Domain → Application → Infrastructure → API) +- Plugin Pattern (dynamic loading, base classes, interface contracts) +- Interceptor Pattern (EF Core SaveChangesInterceptor for change tracking) +- Hub Pattern (SignalR for real-time notifications) +- Service Pattern (dependency injection throughout) +- Repository Pattern (EF Core DbContext as repository) +- Background Service Pattern (JobWorkerService for async processing) + +**Tech Versions:** +- .NET 10.0 (upgraded from .NET 8.0) +- EF Core 10.0 +- C# 13.0 +- Blazor Server (net10.0) +- MAUI (net10.0-ios/android/windows/macos) +- MudBlazor 8.0 + +**Architectural Decisions:** +- Plugin base classes in Infrastructure for reusability +- SignalR for real-time updates (no polling) +- Background service for job processing (no external scheduler) +- Soft deletes with audit trails +- Foreign key RESTRICT to prevent accidental cascades +- Feature-based folder structure in UI + +--- + +## QUICK REFERENCE + +**Database:** +- SQL Server with 47 EF Core migrations +- Auto-timestamps via GETUTCDATE() defaults +- Soft deletes (IsDeleted flag) +- Audit trails (CreatedBy, ModifiedBy, RecordHistory table) + +**Build Process:** +- MSBuild target copies plugin DLLs to `bin/Plugins/` after build +- Plugins: DiunaBI.Plugins.Morska.dll, DiunaBI.Plugins.PedrolloPL.dll + +**SignalR:** +- Hub: `/hubs/entitychanges` +- Broadcasts: `EntityChanged(module, id, operation)` +- Auto-reconnect enabled in UI +- Real-time updates for QueueJobs, Layers, Records + +**Job Queue:** +- Auto-scheduling from layer configs (Type=ImportWorker/ProcessWorker) +- Background processing every 10 seconds +- Retry logic: 30s → 2m → 5m (max 5 retries) +- Priority-based execution (0 = highest) +- Real-time status updates via SignalR + +**Plugins:** +- **Morska:** 4 importers, 12 processors, 1 exporter (~6,566 LOC) +- **PedrolloPL:** 1 importer (P2 data) +- Base classes: BaseDataImporter, BaseDataProcessor, BaseDataExporter +- Dynamic loading from `bin/Plugins/` at startup + +**UI Structure:** +- Feature-based folders: Pages/Layers, Pages/Jobs, Pages/DataInbox +- Separate code-behind for complex logic (.razor.cs files) +- Inline @code for simple pages +- Organized components: Layout/, Auth/ +- Filter state persistence across navigation + +--- + +## FILE PATHS REFERENCE + +**Key Configuration:** +- API: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.API/appsettings.json` +- API Startup: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.API/Program.cs` + +**SignalR:** +- Hub: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.API/Hubs/EntityChangeHub.cs` +- Interceptor: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Infrastructure/Interceptors/EntityChangeInterceptor.cs` +- UI Service: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.UI.Shared/Services/EntityChangeHubService.cs` + +**Job System:** +- Controller: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.API/Controllers/JobsController.cs` +- Scheduler: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Infrastructure/Services/JobSchedulerService.cs` +- Worker: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Infrastructure/Services/JobWorkerService.cs` +- UI Pages: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.UI.Shared/Pages/Jobs/` + +**Plugins:** +- Base Classes: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Infrastructure/Plugins/` +- Morska: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Plugins.Morska/` +- PedrolloPL: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Plugins.PedrolloPL/` + +**Migrations:** +- Latest: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.Infrastructure/Migrations/20251201165810_RecordHistory.cs` + +**UI Components:** +- Pages: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.UI.Shared/Pages/` +- Components: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.UI.Shared/Components/` +- Services: `/Users/mz/Projects/Diuna/DiunaBI/DiunaBI.UI.Shared/Services/` diff --git a/DiunaBI.UI.Shared/Components/AuthGuard.razor b/DiunaBI.UI.Shared/Components/Auth/AuthGuard.razor similarity index 100% rename from DiunaBI.UI.Shared/Components/AuthGuard.razor rename to DiunaBI.UI.Shared/Components/Auth/AuthGuard.razor diff --git a/DiunaBI.UI.Shared/Components/LoginCard.razor b/DiunaBI.UI.Shared/Components/Auth/LoginCard.razor similarity index 100% rename from DiunaBI.UI.Shared/Components/LoginCard.razor rename to DiunaBI.UI.Shared/Components/Auth/LoginCard.razor diff --git a/DiunaBI.UI.Shared/EmptyLayout.razor b/DiunaBI.UI.Shared/Components/Layout/EmptyLayout.razor similarity index 100% rename from DiunaBI.UI.Shared/EmptyLayout.razor rename to DiunaBI.UI.Shared/Components/Layout/EmptyLayout.razor diff --git a/DiunaBI.UI.Shared/MainLayout.razor b/DiunaBI.UI.Shared/Components/Layout/MainLayout.razor similarity index 100% rename from DiunaBI.UI.Shared/MainLayout.razor rename to DiunaBI.UI.Shared/Components/Layout/MainLayout.razor diff --git a/DiunaBI.UI.Shared/Components/Routes.razor b/DiunaBI.UI.Shared/Components/Layout/Routes.razor similarity index 100% rename from DiunaBI.UI.Shared/Components/Routes.razor rename to DiunaBI.UI.Shared/Components/Layout/Routes.razor diff --git a/DiunaBI.UI.Shared/Components/Dashboard.razor b/DiunaBI.UI.Shared/Pages/Dashboard.razor similarity index 100% rename from DiunaBI.UI.Shared/Components/Dashboard.razor rename to DiunaBI.UI.Shared/Pages/Dashboard.razor diff --git a/DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor b/DiunaBI.UI.Shared/Pages/DataInbox/Details.razor similarity index 97% rename from DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor rename to DiunaBI.UI.Shared/Pages/DataInbox/Details.razor index 97b726c..a4d1ba4 100644 --- a/DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor +++ b/DiunaBI.UI.Shared/Pages/DataInbox/Details.razor @@ -2,8 +2,6 @@ @using DiunaBI.UI.Shared.Services @using DiunaBI.Application.DTOModels @using MudBlazor -@inject DataInboxService DataInboxService -@inject NavigationManager NavigationManager diff --git a/DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor.cs b/DiunaBI.UI.Shared/Pages/DataInbox/Details.razor.cs similarity index 86% rename from DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor.cs rename to DiunaBI.UI.Shared/Pages/DataInbox/Details.razor.cs index 3e30298..ffd45af 100644 --- a/DiunaBI.UI.Shared/Pages/DataInboxDetailPage.razor.cs +++ b/DiunaBI.UI.Shared/Pages/DataInbox/Details.razor.cs @@ -1,15 +1,22 @@ using DiunaBI.Application.DTOModels; +using DiunaBI.UI.Shared.Services; using Microsoft.AspNetCore.Components; using MudBlazor; using System.Text; -namespace DiunaBI.UI.Shared.Pages; +namespace DiunaBI.UI.Shared.Pages.DataInbox; -public partial class DataInboxDetailPage : ComponentBase +public partial class Details : ComponentBase { [Parameter] public Guid Id { get; set; } + [Inject] + private DataInboxService DataInboxService { get; set; } = null!; + + [Inject] + private NavigationManager NavigationManager { get; set; } = null!; + [Inject] private ISnackbar Snackbar { get; set; } = null!; diff --git a/DiunaBI.UI.Shared/Components/DataInboxListComponent.razor b/DiunaBI.UI.Shared/Pages/DataInbox/Index.razor similarity index 95% rename from DiunaBI.UI.Shared/Components/DataInboxListComponent.razor rename to DiunaBI.UI.Shared/Pages/DataInbox/Index.razor index 78cc7ea..fb3f281 100644 --- a/DiunaBI.UI.Shared/Components/DataInboxListComponent.razor +++ b/DiunaBI.UI.Shared/Pages/DataInbox/Index.razor @@ -1,4 +1,10 @@ +@page "/datainbox" @using MudBlazor.Internal +@using DiunaBI.Application.DTOModels + +Data Inbox + + } + diff --git a/DiunaBI.UI.Shared/Components/DataInboxListComponent.razor.cs b/DiunaBI.UI.Shared/Pages/DataInbox/Index.razor.cs similarity index 95% rename from DiunaBI.UI.Shared/Components/DataInboxListComponent.razor.cs rename to DiunaBI.UI.Shared/Pages/DataInbox/Index.razor.cs index ed584e2..02fbbf2 100644 --- a/DiunaBI.UI.Shared/Components/DataInboxListComponent.razor.cs +++ b/DiunaBI.UI.Shared/Pages/DataInbox/Index.razor.cs @@ -6,9 +6,9 @@ using DiunaBI.Application.DTOModels.Common; using MudBlazor; using Microsoft.JSInterop; -namespace DiunaBI.UI.Shared.Components; +namespace DiunaBI.UI.Shared.Pages.DataInbox; -public partial class DataInboxListComponent : ComponentBase +public partial class Index : ComponentBase { [Inject] private DataInboxService DataInboxService { get; set; } = default!; [Inject] private ISnackbar Snackbar { get; set; } = default!; diff --git a/DiunaBI.UI.Shared/Pages/DataInboxListPage.razor b/DiunaBI.UI.Shared/Pages/DataInboxListPage.razor deleted file mode 100644 index e0629a4..0000000 --- a/DiunaBI.UI.Shared/Pages/DataInboxListPage.razor +++ /dev/null @@ -1,8 +0,0 @@ -@page "/datainbox" -@using DiunaBI.UI.Shared.Components - -Data Inbox - - - - diff --git a/DiunaBI.UI.Shared/Components/Index.razor b/DiunaBI.UI.Shared/Pages/Index.razor similarity index 100% rename from DiunaBI.UI.Shared/Components/Index.razor rename to DiunaBI.UI.Shared/Pages/Index.razor diff --git a/DiunaBI.UI.Shared/Pages/JobListPage.razor b/DiunaBI.UI.Shared/Pages/JobListPage.razor deleted file mode 100644 index 9f9bc78..0000000 --- a/DiunaBI.UI.Shared/Pages/JobListPage.razor +++ /dev/null @@ -1,8 +0,0 @@ -@page "/jobs" -@using DiunaBI.UI.Shared.Components - -Jobs - - - - diff --git a/DiunaBI.UI.Shared/Pages/JobDetailPage.razor b/DiunaBI.UI.Shared/Pages/Jobs/Details.razor similarity index 100% rename from DiunaBI.UI.Shared/Pages/JobDetailPage.razor rename to DiunaBI.UI.Shared/Pages/Jobs/Details.razor diff --git a/DiunaBI.UI.Shared/Components/JobListComponent.razor b/DiunaBI.UI.Shared/Pages/Jobs/Index.razor similarity index 97% rename from DiunaBI.UI.Shared/Components/JobListComponent.razor rename to DiunaBI.UI.Shared/Pages/Jobs/Index.razor index 4861380..b3235d2 100644 --- a/DiunaBI.UI.Shared/Components/JobListComponent.razor +++ b/DiunaBI.UI.Shared/Pages/Jobs/Index.razor @@ -1,5 +1,11 @@ +@page "/jobs" @using MudBlazor.Internal @using DiunaBI.Domain.Entities +@implements IDisposable + +Jobs + + } + diff --git a/DiunaBI.UI.Shared/Components/JobListComponent.razor.cs b/DiunaBI.UI.Shared/Pages/Jobs/Index.razor.cs similarity index 97% rename from DiunaBI.UI.Shared/Components/JobListComponent.razor.cs rename to DiunaBI.UI.Shared/Pages/Jobs/Index.razor.cs index 0ffdd6d..7d5e26c 100644 --- a/DiunaBI.UI.Shared/Components/JobListComponent.razor.cs +++ b/DiunaBI.UI.Shared/Pages/Jobs/Index.razor.cs @@ -6,9 +6,9 @@ using DiunaBI.Domain.Entities; using MudBlazor; using Microsoft.JSInterop; -namespace DiunaBI.UI.Shared.Components; +namespace DiunaBI.UI.Shared.Pages.Jobs; -public partial class JobListComponent : ComponentBase, IDisposable +public partial class Index : ComponentBase, IDisposable { [Inject] private JobService JobService { get; set; } = default!; [Inject] private EntityChangeHubService HubService { get; set; } = default!; diff --git a/DiunaBI.UI.Shared/Pages/LayerListPage.razor b/DiunaBI.UI.Shared/Pages/LayerListPage.razor deleted file mode 100644 index 84ff3d8..0000000 --- a/DiunaBI.UI.Shared/Pages/LayerListPage.razor +++ /dev/null @@ -1,8 +0,0 @@ -@page "/layers" -@using DiunaBI.UI.Shared.Components - -Layers - - - - \ No newline at end of file diff --git a/DiunaBI.UI.Shared/Pages/LayerDetailPage.razor b/DiunaBI.UI.Shared/Pages/Layers/Details.razor similarity index 99% rename from DiunaBI.UI.Shared/Pages/LayerDetailPage.razor rename to DiunaBI.UI.Shared/Pages/Layers/Details.razor index ad10fd9..d8a0f33 100644 --- a/DiunaBI.UI.Shared/Pages/LayerDetailPage.razor +++ b/DiunaBI.UI.Shared/Pages/Layers/Details.razor @@ -2,10 +2,6 @@ @using DiunaBI.UI.Shared.Services @using DiunaBI.Application.DTOModels @using MudBlazor -@inject LayerService LayerService -@inject JobService JobService -@inject NavigationManager NavigationManager -@inject ISnackbar Snackbar diff --git a/DiunaBI.UI.Shared/Pages/LayerDetailPage.razor.cs b/DiunaBI.UI.Shared/Pages/Layers/Details.razor.cs similarity index 97% rename from DiunaBI.UI.Shared/Pages/LayerDetailPage.razor.cs rename to DiunaBI.UI.Shared/Pages/Layers/Details.razor.cs index ceafab4..16171df 100644 --- a/DiunaBI.UI.Shared/Pages/LayerDetailPage.razor.cs +++ b/DiunaBI.UI.Shared/Pages/Layers/Details.razor.cs @@ -1,11 +1,12 @@ using DiunaBI.Application.DTOModels; +using DiunaBI.UI.Shared.Services; using Microsoft.AspNetCore.Components; using MudBlazor; using System.Reflection; -namespace DiunaBI.UI.Shared.Pages; +namespace DiunaBI.UI.Shared.Pages.Layers; -public partial class LayerDetailPage : ComponentBase +public partial class Details : ComponentBase { [Parameter] public Guid Id { get; set; } @@ -13,6 +14,18 @@ public partial class LayerDetailPage : ComponentBase [Inject] private IDialogService DialogService { get; set; } = null!; + [Inject] + private LayerService LayerService { get; set; } = null!; + + [Inject] + private JobService JobService { get; set; } = null!; + + [Inject] + private NavigationManager NavigationManager { get; set; } = null!; + + [Inject] + private ISnackbar Snackbar { get; set; } = null!; + private LayerDto? layer; private List records = new(); private List displayedColumns = new(); diff --git a/DiunaBI.UI.Shared/Components/LayerListComponent.razor b/DiunaBI.UI.Shared/Pages/Layers/Index.razor similarity index 97% rename from DiunaBI.UI.Shared/Components/LayerListComponent.razor rename to DiunaBI.UI.Shared/Pages/Layers/Index.razor index efbf64e..259793b 100644 --- a/DiunaBI.UI.Shared/Components/LayerListComponent.razor +++ b/DiunaBI.UI.Shared/Pages/Layers/Index.razor @@ -1,5 +1,10 @@ +@page "/layers" @using MudBlazor.Internal @using DiunaBI.Application.DTOModels + +Layers + + - + - } \ No newline at end of file + } + diff --git a/DiunaBI.UI.Shared/Components/LayerListComponent.razor.cs b/DiunaBI.UI.Shared/Pages/Layers/Index.razor.cs similarity index 95% rename from DiunaBI.UI.Shared/Components/LayerListComponent.razor.cs rename to DiunaBI.UI.Shared/Pages/Layers/Index.razor.cs index f21539c..b4f5234 100644 --- a/DiunaBI.UI.Shared/Components/LayerListComponent.razor.cs +++ b/DiunaBI.UI.Shared/Pages/Layers/Index.razor.cs @@ -6,9 +6,9 @@ using DiunaBI.Application.DTOModels.Common; using MudBlazor; using Microsoft.JSInterop; -namespace DiunaBI.UI.Shared.Components; +namespace DiunaBI.UI.Shared.Pages.Layers; -public partial class LayerListComponent : ComponentBase +public partial class Index : ComponentBase { [Inject] private LayerService LayerService { get; set; } = default!; [Inject] private ISnackbar Snackbar { get; set; } = default!; @@ -51,7 +51,7 @@ public partial class LayerListComponent : ComponentBase filterRequest.Page = 1; await LoadLayers(); } - + private async Task OnPageChanged(int page) { filterRequest.Page = page; @@ -89,4 +89,4 @@ public partial class LayerListComponent : ComponentBase var url = NavigationManager.ToAbsoluteUri($"/layers/{layer.Id}").ToString(); await JSRuntime.InvokeVoidAsync("open", url, "_blank"); } -} \ No newline at end of file +} diff --git a/DiunaBI.UI.Shared/Pages/LoginPage.razor b/DiunaBI.UI.Shared/Pages/Login.razor similarity index 100% rename from DiunaBI.UI.Shared/Pages/LoginPage.razor rename to DiunaBI.UI.Shared/Pages/Login.razor diff --git a/DiunaBI.UI.Shared/_Imports.razor b/DiunaBI.UI.Shared/_Imports.razor index 69701a9..2594a36 100644 --- a/DiunaBI.UI.Shared/_Imports.razor +++ b/DiunaBI.UI.Shared/_Imports.razor @@ -8,5 +8,7 @@ @using Microsoft.JSInterop @using DiunaBI.UI.Shared @using DiunaBI.UI.Shared.Components +@using DiunaBI.UI.Shared.Components.Layout +@using DiunaBI.UI.Shared.Components.Auth @using DiunaBI.Application.DTOModels @using MudBlazor \ No newline at end of file