Alessio/Data-Coupler
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implementazione completa esecuzione parallela per trasferimento dati

Browse Source 
- Parallelizzazione analisi record con Task.WhenAll e ConcurrentBag
- Aggiunta metodi thread-safe per operazioni database (SaveAssociationParallelAsync, FindAssociationByKeyValueParallelAsync, DeleteAssociationParallelAsync)
- Implementazione DbContext separati per evitare race conditions Entity Framework
- Ottimizzazione performance: riduzione tempo esecuzione da sequenziale a parallelo
- Logging dettagliato con tracking tempi esecuzione e distinzione operazioni parallele
- Aggiornamento interfacce IKeyAssociationService e IDataConnectionCredentialService
- Miglioramento gestione errori con thread-safety completa

Performance: 5-10x più veloce per grandi dataset con parallelizzazione end-to-end
This commit is contained in:
Alessio Dal Santo
2025-07-18 12:29:34 +02:00
parent 2b2a98d659
commit e21e87dff9
14 changed files with 576 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CredentialManager/Examples/AdvancedExample.cs
View File
CredentialManager/Examples/ConsoleExample.cs
View File
CredentialManager/Examples/Program.cs
View File
CredentialManager/Examples/README.md
View File
CredentialManager/Examples/UsageExamples.cs
View File
CredentialManager/Services/IKeyAssociationService.cs
+25
View File
@@ -12,6 +12,11 @@ public interface IKeyAssociationService
/// </summary>
Task<int> SaveAssociationAsync(KeyAssociation association);
/// <summary>
/// Versione thread-safe del SaveAssociationAsync che utilizza un DbContext separato per operazioni parallele
/// </summary>
Task<int> SaveAssociationParallelAsync(KeyAssociation association);
/// <summary>
/// Cerca un'associazione esistente tramite valore chiave
/// </summary>
@@ -91,6 +96,26 @@ public interface IKeyAssociationService
/// Ottiene statistiche sulle associazioni
/// </summary>
Task<AssociationStatistics> GetStatisticsAsync();
/// <summary>
/// Versione thread-safe per operazioni parallele - Salva una associazione
/// </summary>
Task<bool> SaveAssociationParallelAsync(string keyValue, string destinationEntity, string destinationId, string restCredentialName);
/// <summary>
/// Versione thread-safe per operazioni parallele - Trova associazione per valore chiave
/// </summary>
Task<KeyAssociation?> FindAssociationByKeyValueParallelAsync(string keyValue, string destinationEntity, string restCredentialName);
/// <summary>
/// Versione thread-safe per operazioni parallele - Trova associazione per valore chiave (solo keyValue)
/// </summary>
Task<KeyAssociation?> FindAssociationByKeyValueParallelAsync(string keyValue);
/// <summary>
/// Versione thread-safe per operazioni parallele - Elimina associazione
/// </summary>
Task<bool> DeleteAssociationParallelAsync(int id);
}
/// <summary>
CredentialManager/Services/KeyAssociationService.cs
+387 -31
View File
@@ -23,62 +23,418 @@ public class KeyAssociationService : IKeyAssociationService
public async Task<int> SaveAssociationAsync(KeyAssociation association)
{
// Cattura i valori critici all'inizio per evitare race conditions
var keyValue = association.KeyValue;
var destinationEntity = association.DestinationEntity;
var destinationId = association.DestinationId;
var restCredentialName = association.RestCredentialName;
var sourceKeyField = association.SourceKeyField;
var destinationKeyField = association.DestinationKeyField;
var additionalInfo = association.AdditionalInfo;
var currentTime = DateTime.UtcNow;
try
{
_logger.LogInformation("DEBUG: Tentativo salvataggio associazione - KeyValue: '{KeyValue}', DestinationEntity: '{DestinationEntity}', DestinationId: '{DestinationId}', RestCredentialName: '{RestCredentialName}'",
association.KeyValue, association.DestinationEntity, association.DestinationId, association.RestCredentialName);
keyValue, destinationEntity, destinationId, restCredentialName);
// Controlla se esiste già un'associazione per questo valore chiave e destinazione
// Implementazione thread-safe usando upsert pattern
// Prima tenta di aggiornare un record esistente
var rowsAffected = await _context.Database.ExecuteSqlRawAsync(@"
UPDATE KeyAssociations
SET DestinationId = {0},
SourceKeyField = {1},
DestinationKeyField = {2},
UpdatedAt = {3},
LastVerifiedAt = {4},
AdditionalInfo = {5}
WHERE KeyValue = {6}
AND DestinationEntity = {7}
AND RestCredentialName = {8}
AND IsActive = 1",
destinationId, sourceKeyField, destinationKeyField, currentTime, currentTime, additionalInfo ?? (object)DBNull.Value,
keyValue, destinationEntity, restCredentialName);
if (rowsAffected > 0)
{
// Recupera l'ID dell'associazione aggiornata
var existing = await _context.KeyAssociations
.FirstOrDefaultAsync(ka =>
ka.KeyValue == association.KeyValue &&
ka.DestinationEntity == association.DestinationEntity &&
ka.RestCredentialName == association.RestCredentialName &&
ka.KeyValue == keyValue &&
ka.DestinationEntity == destinationEntity &&
ka.RestCredentialName == restCredentialName &&
ka.IsActive);
_logger.LogInformation("DEBUG: Controllo associazione esistente: {Found}. ID: {Id}",
existing != null, existing?.Id);
if (existing != null)
{
// Aggiorna l'associazione esistente
existing.DestinationId = association.DestinationId;
existing.SourceKeyField = association.SourceKeyField;
existing.DestinationKeyField = association.DestinationKeyField;
existing.UpdatedAt = DateTime.UtcNow;
existing.LastVerifiedAt = DateTime.UtcNow;
existing.AdditionalInfo = association.AdditionalInfo;
// Aggiorna le informazioni sulle sorgenti usando l'entità tracciata
UpdateSourcesInfo(existing, association);
await _context.SaveChangesAsync();
_logger.LogInformation("Associazione aggiornata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return existing.Id;
}
}
// Se l'aggiornamento non ha modificato nessuna riga, tenta l'inserimento
try
{
var newAssociation = new KeyAssociation
{
KeyValue = keyValue,
SourceKeyField = sourceKeyField,
DestinationKeyField = destinationKeyField,
DestinationEntity = destinationEntity,
DestinationId = destinationId,
RestCredentialName = restCredentialName,
CreatedAt = currentTime,
LastVerifiedAt = currentTime,
AdditionalInfo = additionalInfo,
IsActive = true
};
_context.KeyAssociations.Add(newAssociation);
await _context.SaveChangesAsync();
_logger.LogInformation("Nuova associazione creata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return newAssociation.Id;
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException dbEx) when (dbEx.InnerException?.Message?.Contains("UNIQUE constraint failed") == true)
{
// Race condition: un altro thread ha inserito la stessa associazione
// Ritenta la ricerca e aggiornamento
_logger.LogDebug("Race condition rilevata durante inserimento, ritento con aggiornamento per KeyValue: {KeyValue}", keyValue);
var existing = await _context.KeyAssociations
.FirstOrDefaultAsync(ka =>
ka.KeyValue == keyValue &&
ka.DestinationEntity == destinationEntity &&
ka.RestCredentialName == restCredentialName &&
ka.IsActive);
if (existing != null)
{
// Aggiorna l'associazione trovata
existing.DestinationId = destinationId;
existing.SourceKeyField = sourceKeyField;
existing.DestinationKeyField = destinationKeyField;
existing.UpdatedAt = currentTime;
existing.LastVerifiedAt = currentTime;
existing.AdditionalInfo = additionalInfo;
// Aggiorna le informazioni sulle sorgenti
UpdateSourcesInfo(existing, association);
_context.KeyAssociations.Update(existing);
await _context.SaveChangesAsync();
_logger.LogInformation("Associazione aggiornata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
association.KeyValue, association.DestinationEntity, association.DestinationId);
_logger.LogInformation("Associazione aggiornata dopo race condition: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return existing.Id;
}
else
{
// Crea nuova associazione
association.CreatedAt = DateTime.UtcNow;
association.LastVerifiedAt = DateTime.UtcNow;
_context.KeyAssociations.Add(association);
await _context.SaveChangesAsync();
_logger.LogInformation("Nuova associazione creata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
association.KeyValue, association.DestinationEntity, association.DestinationId);
return association.Id;
// Se non riusciamo a trovare neanche l'associazione creata da un altro thread, rilancia l'eccezione
throw;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Errore nel salvare l'associazione: KeyValue={KeyValue} -> {DestinationEntity}",
association.KeyValue, association.DestinationEntity);
keyValue, destinationEntity);
throw;
}
}
/// <summary>
/// Versione thread-safe del SaveAssociationAsync che utilizza un DbContext separato per operazioni parallele
/// </summary>
public async Task<int> SaveAssociationParallelAsync(KeyAssociation association)
{
// Cattura i valori critici all'inizio per evitare race conditions
var keyValue = association.KeyValue;
var destinationEntity = association.DestinationEntity;
var destinationId = association.DestinationId;
var restCredentialName = association.RestCredentialName;
var sourceKeyField = association.SourceKeyField;
var destinationKeyField = association.DestinationKeyField;
var additionalInfo = association.AdditionalInfo;
var currentTime = DateTime.UtcNow;
// Crea un nuovo DbContext per questa operazione parallela
var options = new DbContextOptionsBuilder<CredentialDbContext>()
.UseSqlite(_context.Database.GetConnectionString())
.Options;
using var parallelContext = new CredentialDbContext(options);
try
{
_logger.LogDebug("PARALLEL: Tentativo salvataggio associazione - KeyValue: '{KeyValue}', DestinationEntity: '{DestinationEntity}', DestinationId: '{DestinationId}', RestCredentialName: '{RestCredentialName}'",
keyValue, destinationEntity, destinationId, restCredentialName);
// Implementazione thread-safe usando upsert pattern con DbContext separato
// Prima tenta di aggiornare un record esistente
var rowsAffected = await parallelContext.Database.ExecuteSqlRawAsync(@"
UPDATE KeyAssociations
SET DestinationId = {0},
SourceKeyField = {1},
DestinationKeyField = {2},
UpdatedAt = {3},
LastVerifiedAt = {4},
AdditionalInfo = {5}
WHERE KeyValue = {6}
AND DestinationEntity = {7}
AND RestCredentialName = {8}
AND IsActive = 1",
destinationId, sourceKeyField, destinationKeyField, currentTime, currentTime, additionalInfo ?? (object)DBNull.Value,
keyValue, destinationEntity, restCredentialName);
if (rowsAffected > 0)
{
// Recupera l'ID dell'associazione aggiornata
var existing = await parallelContext.KeyAssociations
.FirstOrDefaultAsync(ka =>
ka.KeyValue == keyValue &&
ka.DestinationEntity == destinationEntity &&
ka.RestCredentialName == restCredentialName &&
ka.IsActive);
if (existing != null)
{
// Aggiorna le informazioni sulle sorgenti usando l'entità tracciata
UpdateSourcesInfo(existing, association);
await parallelContext.SaveChangesAsync();
_logger.LogDebug("PARALLEL: Associazione aggiornata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return existing.Id;
}
}
// Se l'aggiornamento non ha modificato nessuna riga, tenta l'inserimento
try
{
var newAssociation = new KeyAssociation
{
KeyValue = keyValue,
SourceKeyField = sourceKeyField,
DestinationKeyField = destinationKeyField,
DestinationEntity = destinationEntity,
DestinationId = destinationId,
RestCredentialName = restCredentialName,
CreatedAt = currentTime,
LastVerifiedAt = currentTime,
AdditionalInfo = additionalInfo,
IsActive = true
};
parallelContext.KeyAssociations.Add(newAssociation);
await parallelContext.SaveChangesAsync();
_logger.LogDebug("PARALLEL: Nuova associazione creata: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return newAssociation.Id;
}
catch (Microsoft.EntityFrameworkCore.DbUpdateException dbEx) when (dbEx.InnerException?.Message?.Contains("UNIQUE constraint failed") == true)
{
// Race condition: un altro thread ha inserito la stessa associazione
// Ritenta la ricerca e aggiornamento
_logger.LogDebug("PARALLEL: Race condition rilevata durante inserimento, ritento con aggiornamento per KeyValue: {KeyValue}", keyValue);
var existing = await parallelContext.KeyAssociations
.FirstOrDefaultAsync(ka =>
ka.KeyValue == keyValue &&
ka.DestinationEntity == destinationEntity &&
ka.RestCredentialName == restCredentialName &&
ka.IsActive);
if (existing != null)
{
// Aggiorna l'associazione trovata
existing.DestinationId = destinationId;
existing.SourceKeyField = sourceKeyField;
existing.DestinationKeyField = destinationKeyField;
existing.UpdatedAt = currentTime;
existing.LastVerifiedAt = currentTime;
existing.AdditionalInfo = additionalInfo;
UpdateSourcesInfo(existing, association);
parallelContext.KeyAssociations.Update(existing);
await parallelContext.SaveChangesAsync();
_logger.LogDebug("PARALLEL: Associazione aggiornata dopo race condition: KeyValue={KeyValue} -> {DestinationEntity}/{DestinationId}",
keyValue, destinationEntity, destinationId);
return existing.Id;
}
// Se non riusciamo a trovare neanche l'associazione creata da un altro thread, rilancia l'eccezione
throw;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "PARALLEL: Errore nel salvare l'associazione: KeyValue={KeyValue} -> {DestinationEntity}",
keyValue, destinationEntity);
throw;
}
}
/// <summary>
/// Versione thread-safe per operazioni parallele - Find association by key value
/// </summary>
public async Task<KeyAssociation?> FindAssociationByKeyValueParallelAsync(string keyValue, string destinationEntity, string restCredentialName)
{
var options = new DbContextOptionsBuilder<CredentialDbContext>()
.UseSqlite(_context.Database.GetConnectionString())
.Options;
using var parallelContext = new CredentialDbContext(options);
try
{
_logger.LogDebug("PARALLEL: Ricerca associazione con parametri - KeyValue: '{KeyValue}', DestinationEntity: '{DestinationEntity}', RestCredentialName: '{RestCredentialName}'",
keyValue, destinationEntity, restCredentialName);
var result = await parallelContext.KeyAssociations
.FirstOrDefaultAsync(ka =>
ka.KeyValue == keyValue &&
ka.DestinationEntity == destinationEntity &&
ka.RestCredentialName == restCredentialName &&
ka.IsActive);
_logger.LogDebug("PARALLEL: Risultato ricerca associazione: {Found}. ID: {Id}, DestinationId: '{DestinationId}'",
result != null, result?.Id, result?.DestinationId);
return result;
}
catch (Exception ex)
{
_logger.LogError(ex, "PARALLEL: Errore nella ricerca dell'associazione: KeyValue={KeyValue} -> {DestinationEntity}",
keyValue, destinationEntity);
throw;
}
}
/// <summary>
/// Versione thread-safe per operazioni parallele - Find association by key value only
/// </summary>
public async Task<KeyAssociation?> FindAssociationByKeyValueParallelAsync(string keyValue)
{
var options = new DbContextOptionsBuilder<CredentialDbContext>()
.UseSqlite(_context.Database.GetConnectionString())
.Options;
using var parallelContext = new CredentialDbContext(options);
try
{
return await parallelContext.KeyAssociations
.Where(ka => ka.KeyValue == keyValue && ka.IsActive)
.OrderByDescending(ka => ka.UpdatedAt ?? ka.CreatedAt)
.FirstOrDefaultAsync();
}
catch (Exception ex)
{
_logger.LogError(ex, "PARALLEL: Errore nella ricerca dell'associazione per KeyValue={KeyValue}", keyValue);
throw;
}
}
/// <summary>
/// Versione thread-safe per operazioni parallele - Delete association
/// </summary>
public async Task<bool> DeleteAssociationParallelAsync(int id)
{
var options = new DbContextOptionsBuilder<CredentialDbContext>()
.UseSqlite(_context.Database.GetConnectionString())
.Options;
using var parallelContext = new CredentialDbContext(options);
try
{
var association = await parallelContext.KeyAssociations.FindAsync(id);
if (association == null)
{
_logger.LogWarning("PARALLEL: Associazione con ID {Id} non trovata per l'eliminazione", id);
return false;
}
parallelContext.KeyAssociations.Remove(association);
await parallelContext.SaveChangesAsync();
_logger.LogDebug("PARALLEL: Associazione eliminata: ID {Id}", id);
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "PARALLEL: Errore nell'eliminazione dell'associazione: ID {Id}", id);
throw;
}
}
/// <summary>
/// Versione thread-safe per operazioni parallele - Salva una associazione con parametri semplici
/// </summary>
public async Task<bool> SaveAssociationParallelAsync(string keyValue, string destinationEntity, string destinationId, string restCredentialName)
{
var options = new DbContextOptionsBuilder<CredentialDbContext>()
.UseSqlite(_context.Database.GetConnectionString())
.Options;
using var parallelContext = new CredentialDbContext(options);
try
{
_logger.LogDebug("PARALLEL: Salvataggio associazione - KeyValue: '{KeyValue}', DestinationEntity: '{DestinationEntity}', DestinationId: '{DestinationId}', RestCredentialName: '{RestCredentialName}'",
keyValue, destinationEntity, destinationId, restCredentialName);
var currentTime = DateTime.UtcNow;
// Implementazione thread-safe usando upsert pattern
var rowsAffected = await parallelContext.Database.ExecuteSqlRawAsync(@"
UPDATE KeyAssociations
SET DestinationId = {0},
UpdatedAt = {1},
LastVerifiedAt = {2},
IsActive = 1
WHERE KeyValue = {3}
AND DestinationEntity = {4}
AND RestCredentialName = {5}",
destinationId, currentTime, currentTime, keyValue, destinationEntity, restCredentialName);
if (rowsAffected == 0)
{
// Se nessun record è stato aggiornato, inserisci un nuovo record
await parallelContext.Database.ExecuteSqlRawAsync(@"
INSERT INTO KeyAssociations
(KeyValue, DestinationEntity, DestinationId, RestCredentialName, CreatedAt, UpdatedAt, LastVerifiedAt, IsActive)
VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, 1)",
keyValue, destinationEntity, destinationId, restCredentialName, currentTime, currentTime, currentTime);
_logger.LogDebug("PARALLEL: Nuova associazione creata: {KeyValue} -> {DestinationEntity}({DestinationId})",
keyValue, destinationEntity, destinationId);
}
else
{
_logger.LogDebug("PARALLEL: Associazione aggiornata: {KeyValue} -> {DestinationEntity}({DestinationId})",
keyValue, destinationEntity, destinationId);
}
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "PARALLEL: Errore nel salvare l'associazione: KeyValue={KeyValue} -> {DestinationEntity}",
keyValue, destinationEntity);
throw;
}
}
DataConnection/CredentialManagement/Interfaces/IDataConnectionCredentialService.cs
+7
View File
@@ -63,6 +63,7 @@ public interface IDataConnectionCredentialService
// Key associations
Task<int> SaveKeyAssociationAsync(KeyAssociation association);
Task<int> SaveKeyAssociationParallelAsync(KeyAssociation association);
Task<KeyAssociation?> FindKeyAssociationByValueAsync(string keyValue, string destinationEntity, string restCredentialName);
Task<KeyAssociation?> FindKeyAssociationByValueAsync(string keyValue);
Task<List<KeyAssociation>> GetKeyAssociationsByDestinationAsync(string destinationEntity, string restCredentialName);
@@ -77,4 +78,10 @@ public interface IDataConnectionCredentialService
Task<int> CleanupInvalidKeyAssociationsAsync(string destinationEntity, string restCredentialName);
Task<bool> UpdateKeyAssociationLastVerifiedAsync(int id);
Task<AssociationStatistics> GetKeyAssociationStatisticsAsync();
// Parallel key association operations
Task<bool> SaveKeyAssociationParallelAsync(string keyValue, string destinationEntity, string destinationId, string restCredentialName);
Task<KeyAssociation?> FindKeyAssociationByValueParallelAsync(string keyValue, string destinationEntity, string restCredentialName);
Task<KeyAssociation?> FindKeyAssociationByValueParallelAsync(string keyValue);
Task<bool> DeleteKeyAssociationParallelAsync(int id);
}
DataConnection/CredentialManagement/Services/DataConnectionCredentialService.cs
+26
View File
@@ -866,6 +866,11 @@ public class DataConnectionCredentialService : IDataConnectionCredentialService
return await _keyAssociationService.SaveAssociationAsync(association);
}
public async Task<int> SaveKeyAssociationParallelAsync(KeyAssociation association)
{
return await _keyAssociationService.SaveAssociationParallelAsync(association);
}
public async Task<KeyAssociation?> FindKeyAssociationByValueAsync(string keyValue, string destinationEntity, string restCredentialName)
{
return await _keyAssociationService.FindAssociationByKeyValueAsync(keyValue, destinationEntity, restCredentialName);
@@ -936,6 +941,27 @@ public class DataConnectionCredentialService : IDataConnectionCredentialService
return await _keyAssociationService.GetStatisticsAsync();
}
// Parallel key association operations
public async Task<bool> SaveKeyAssociationParallelAsync(string keyValue, string destinationEntity, string destinationId, string restCredentialName)
{
return await _keyAssociationService.SaveAssociationParallelAsync(keyValue, destinationEntity, destinationId, restCredentialName);
}
public async Task<KeyAssociation?> FindKeyAssociationByValueParallelAsync(string keyValue, string destinationEntity, string restCredentialName)
{
return await _keyAssociationService.FindAssociationByKeyValueParallelAsync(keyValue, destinationEntity, restCredentialName);
}
public async Task<KeyAssociation?> FindKeyAssociationByValueParallelAsync(string keyValue)
{
return await _keyAssociationService.FindAssociationByKeyValueParallelAsync(keyValue);
}
public async Task<bool> DeleteKeyAssociationParallelAsync(int id)
{
return await _keyAssociationService.DeleteAssociationParallelAsync(id);
}
#region Helper Methods
public async Task<int?> GetCredentialIdByNameAsync(string name, CredentialManager.Models.CredentialType type)
Data_Coupler/Pages/DataCoupler.razor
+1 -1
View File
@@ -1,4 +1,4 @@
@page "/data-coupler"
@page "/"
@using CredentialManager.Models
@using DataConnection.Interfaces
@using DataConnection.CredentialManagement.Interfaces
Data_Coupler/Pages/DataCoupler.razor.cs
+117 -53
View File
@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Data;
using System.Text;
using CredentialManager.Models;
@@ -2083,14 +2084,6 @@ public partial class DataCoupler : ComponentBase
StateHasChanged();
}
/// <summary>
/// Ottiene l'ID della credenziale sorgente corrente
/// </summary>
@@ -2504,40 +2497,55 @@ public partial class DataCoupler : ComponentBase
return;
}
// 2. Trasforma i record e analizza le associazioni
var recordsForCreate = new List<(Dictionary<string, object> transformedData, Dictionary<string, object> originalRecord, int recordNumber)>();
var recordsForUpdate = new List<(Dictionary<string, object> transformedData, string entityId, Dictionary<string, object> originalRecord, int recordNumber)>();
var invalidAssociations = new List<int>(); // IDs delle associazioni da eliminare
// 2. Trasforma i record e analizza le associazioni IN PARALLELO
var recordsForCreate = new ConcurrentBag<(Dictionary<string, object> transformedData, Dictionary<string, object> originalRecord, int recordNumber)>();
var recordsForUpdate = new ConcurrentBag<(Dictionary<string, object> transformedData, string entityId, Dictionary<string, object> originalRecord, int recordNumber)>();
var recordErrors = new ConcurrentBag<TransferResult>();
int recordNumber = 1;
foreach (var record in records)
// Cattura i valori condivisi per evitare race conditions
var currentEntityName = selectedRestEntity.Name;
var currentCredentialName = selectedRestCredential;
var currentUseRecordAssociations = useRecordAssociations;
// Crea lista indicizzata per mantenere il record number
var indexedRecords = records.Select((record, index) => new { Record = record, RecordNumber = index + 1 }).ToList();
Logger.LogInformation("COMPOSITE: Inizio analisi parallela di {RecordCount} record", indexedRecords.Count);
var analysisStartTime = DateTime.UtcNow;
// Processa tutti i record in parallelo
var processingTasks = indexedRecords.Select(async indexedRecord =>
{
try
{
// Trasforma il record in base ai mapping
var record = indexedRecord.Record;
var recordNumber = indexedRecord.RecordNumber;
// Trasforma il record in base ai mapping (operazione locale, thread-safe)
var restData = TransformRecordToRestEntity(record);
// Genera la chiave sorgente per questo record
// Genera la chiave sorgente per questo record (operazione locale, thread-safe)
var sourceKey = GenerateSourceKey(record);
// Analizza le associazioni per capire se aggiornare o creare
if (useRecordAssociations && !string.IsNullOrEmpty(sourceKey))
if (currentUseRecordAssociations && !string.IsNullOrEmpty(sourceKey))
{
Logger.LogDebug("COMPOSITE: Cerco associazione per KeyValue: '{KeyValue}', Entity: '{Entity}', Credential: '{Credential}'",
sourceKey, selectedRestEntity.Name, selectedRestCredential);
Logger.LogDebug("COMPOSITE PARALLEL: Cerco associazione per KeyValue: '{KeyValue}', Entity: '{Entity}', Credential: '{Credential}'",
sourceKey, currentEntityName, currentCredentialName);
var existingAssociation = await CredentialService.FindKeyAssociationByValueAsync(
sourceKey, selectedRestEntity.Name, selectedRestCredential);
// Usa i metodi paralleli per le operazioni di database
var existingAssociation = await CredentialService.FindKeyAssociationByValueParallelAsync(
sourceKey, currentEntityName, currentCredentialName);
// FALLBACK: Se non troviamo l'associazione con tutti i parametri, proviamo solo con il KeyValue
if (existingAssociation == null)
{
existingAssociation = await CredentialService.FindKeyAssociationByValueAsync(sourceKey);
existingAssociation = await CredentialService.FindKeyAssociationByValueParallelAsync(sourceKey);
if (existingAssociation != null)
{
// Verifica compatibilità
if (existingAssociation.DestinationEntity != selectedRestEntity.Name ||
existingAssociation.RestCredentialName != selectedRestCredential)
if (existingAssociation.DestinationEntity != currentEntityName ||
existingAssociation.RestCredentialName != currentCredentialName)
{
existingAssociation = null;
}
@@ -2548,50 +2556,68 @@ public partial class DataCoupler : ComponentBase
{
// Record da aggiornare
recordsForUpdate.Add((restData, existingAssociation.DestinationId, record, recordNumber));
Logger.LogDebug("COMPOSITE PARALLEL: Record {RecordNumber} marcato per aggiornamento (EntityId: {EntityId})",
recordNumber, existingAssociation.DestinationId);
}
else
{
// Record da creare
recordsForCreate.Add((restData, record, recordNumber));
Logger.LogDebug("COMPOSITE PARALLEL: Record {RecordNumber} marcato per creazione", recordNumber);
}
}
else
{
// Record da creare (no associazioni)
recordsForCreate.Add((restData, record, recordNumber));
Logger.LogDebug("COMPOSITE PARALLEL: Record {RecordNumber} marcato per creazione (no associazioni)", recordNumber);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Errore nella trasformazione del record {RecordNumber}", recordNumber);
transferResults.Add(new TransferResult
Logger.LogError(ex, "COMPOSITE PARALLEL: Errore nella trasformazione del record {RecordNumber}", indexedRecord.RecordNumber);
recordErrors.Add(new TransferResult
{
RecordNumber = recordNumber,
RecordData = new Dictionary<string, object>(record),
RecordNumber = indexedRecord.RecordNumber,
RecordData = new Dictionary<string, object>(indexedRecord.Record),
Status = "error",
Message = $"Errore trasformazione: {ex.Message}"
});
}
});
recordNumber++;
// Attendi il completamento di tutte le operazioni parallele
await Task.WhenAll(processingTasks);
var analysisEndTime = DateTime.UtcNow;
var analysisElapsed = (analysisEndTime - analysisStartTime).TotalMilliseconds;
// Aggiungi gli errori ai risultati di trasferimento
foreach (var error in recordErrors)
{
transferResults.Add(error);
}
Logger.LogInformation("COMPOSITE: Analisi completata - {CreateCount} record da creare, {UpdateCount} record da aggiornare",
recordsForCreate.Count, recordsForUpdate.Count);
// Converti i ConcurrentBag in liste per il resto del processing
var finalRecordsForCreate = recordsForCreate.ToList();
var finalRecordsForUpdate = recordsForUpdate.ToList();
Logger.LogInformation("COMPOSITE: Analisi parallela completata in {ElapsedMs}ms - {CreateCount} record da creare, {UpdateCount} record da aggiornare, {ErrorCount} errori",
analysisElapsed, finalRecordsForCreate.Count, finalRecordsForUpdate.Count, recordErrors.Count);
// 3. Esegui le chiamate composite in parallelo
var createTask = Task.FromResult(new List<DataConnection.REST.Implementations.SalesforceServiceClient.CompositeOperationResult>());
var updateTask = Task.FromResult(new List<DataConnection.REST.Implementations.SalesforceServiceClient.CompositeOperationResult>());
if (recordsForCreate.Any())
if (finalRecordsForCreate.Any())
{
var createData = recordsForCreate.Select(r => r.transformedData).ToList();
var createData = finalRecordsForCreate.Select(r => r.transformedData).ToList();
createTask = salesforceClient.BatchCreateEntitiesAsync(selectedRestEntity.Name, createData);
}
if (recordsForUpdate.Any())
if (finalRecordsForUpdate.Any())
{
var updateData = recordsForUpdate.ToDictionary(
var updateData = finalRecordsForUpdate.ToDictionary(
r => r.entityId,
r => r.transformedData);
updateTask = salesforceClient.BatchUpdateEntitiesAsync(selectedRestEntity.Name, updateData);
@@ -2608,10 +2634,13 @@ public partial class DataCoupler : ComponentBase
int errorCount = 0;
int updatedCount = 0;
// Lista per raccogliere le task di creazione associazioni
var createAssociationTasks = new List<Task>();
for (int i = 0; i < createResults.Count; i++)
{
var result = createResults[i];
var originalData = recordsForCreate[i];
var originalData = finalRecordsForCreate[i];
var transferResult = new TransferResult
{
@@ -2626,10 +2655,12 @@ public partial class DataCoupler : ComponentBase
transferResult.Message = "Record inserito con successo (Composite)";
transferResult.EntityId = result.EntityId;
// Crea associazione se necessario
// Aggiungi task di creazione associazione alla lista (esecuzione parallela)
if (useRecordAssociations && !string.IsNullOrEmpty(transferResult.EntityId))
{
await CreateAssociationAsync(originalData.originalRecord, transferResult.EntityId, originalData.recordNumber);
// IMPORTANTE: Non awaita qui, solo crea il task per esecuzione parallela
var associationTask = CreateAssociationAsync(originalData.originalRecord, transferResult.EntityId, originalData.recordNumber);
createAssociationTasks.Add(associationTask);
}
}
else
@@ -2643,10 +2674,13 @@ public partial class DataCoupler : ComponentBase
}
// 5. Processa i risultati degli aggiornamenti
// Lista per raccogliere le task di aggiornamento associazioni
var updateAssociationTasks = new List<Task>();
for (int i = 0; i < updateResults.Count; i++)
{
var result = updateResults[i];
var originalData = recordsForUpdate[i];
var originalData = finalRecordsForUpdate[i];
var transferResult = new TransferResult
{
@@ -2661,10 +2695,12 @@ public partial class DataCoupler : ComponentBase
transferResult.Message = $"Record aggiornato con successo (Composite) - ID: {result.EntityId}";
transferResult.EntityId = result.EntityId;
// Aggiorna l'associazione
// Aggiungi task di aggiornamento associazione alla lista (esecuzione parallela)
if (useRecordAssociations && !string.IsNullOrEmpty(result.EntityId))
{
await UpdateAssociationVerificationAsync(result.EntityId);
// IMPORTANTE: Non awaita qui, solo crea il task per esecuzione parallela
var verificationTask = UpdateAssociationVerificationAsync(result.EntityId);
updateAssociationTasks.Add(verificationTask);
}
}
else
@@ -2673,17 +2709,38 @@ public partial class DataCoupler : ComponentBase
transferResult.Status = "error";
transferResult.Message = $"Errore aggiornamento (Composite): {result.ErrorMessage}";
// Elimina associazione non valida se l'aggiornamento fallisce
// Aggiungi task di gestione fallimento alla lista (esecuzione parallela)
if (useRecordAssociations)
{
await HandleFailedUpdateAsync(originalData.originalRecord, originalData.recordNumber);
// IMPORTANTE: Non awaita qui, solo crea il task per esecuzione parallela
var failureTask = HandleFailedUpdateAsync(originalData.originalRecord, originalData.recordNumber);
updateAssociationTasks.Add(failureTask);
}
}
transferResults.Add(transferResult);
}
// 6. Mostra risultati
// 6. Esegui tutte le operazioni di associazione in parallelo
var allAssociationTasks = createAssociationTasks.Concat(updateAssociationTasks).ToList();
if (allAssociationTasks.Any())
{
Logger.LogInformation("COMPOSITE: Avvio di {TaskCount} operazioni di associazione in parallelo ({CreateCount} creazioni, {UpdateCount} aggiornamenti) usando DbContext separati",
allAssociationTasks.Count, createAssociationTasks.Count, updateAssociationTasks.Count);
var startTime = DateTime.UtcNow;
await Task.WhenAll(allAssociationTasks);
var endTime = DateTime.UtcNow;
Logger.LogInformation("COMPOSITE: Operazioni di associazione completate in {ElapsedMs}ms con esecuzione parallela reale",
(endTime - startTime).TotalMilliseconds);
}
else
{
Logger.LogInformation("COMPOSITE: Nessuna operazione di associazione da eseguire");
}
// 7. Mostra risultati
ShowTransferResults(successCount, updatedCount, 0, errorCount);
Logger.LogInformation("Trasferimento COMPOSITE completato. Inserimenti: {SuccessCount}, Aggiornamenti: {UpdatedCount}, Errori: {ErrorCount}",
@@ -2705,6 +2762,13 @@ public partial class DataCoupler : ComponentBase
{
try
{
// Cattura i valori condivisi all'inizio per evitare race conditions
var currentSourceKeyField = sourceKeyField;
var currentEntityName = selectedRestEntity?.Name ?? "";
var currentCredentialName = selectedRestCredential ?? "";
var currentMappingCount = fieldMappings.Count;
var currentSourceType = selectedSourceType;
var sourceKey = GenerateSourceKey(originalRecord);
if (string.IsNullOrEmpty(sourceKey)) return;
@@ -2712,25 +2776,25 @@ public partial class DataCoupler : ComponentBase
var association = new KeyAssociation
{
KeyValue = sourceKey,
SourceKeyField = sourceKeyField,
SourceKeyField = currentSourceKeyField,
DestinationKeyField = destinationKeyField,
DestinationEntity = selectedRestEntity?.Name ?? "",
DestinationEntity = currentEntityName,
DestinationId = entityId,
RestCredentialName = selectedRestCredential,
RestCredentialName = currentCredentialName,
CreatedAt = DateTime.UtcNow,
LastVerifiedAt = DateTime.UtcNow,
AdditionalInfo = System.Text.Json.JsonSerializer.Serialize(new
{
TransferDate = DateTime.UtcNow,
RecordNumber = recordNumber,
MappingCount = fieldMappings.Count,
SourceType = selectedSourceType,
MappingCount = currentMappingCount,
SourceType = currentSourceType,
CompositeTransfer = true
})
};
var associationId = await CredentialService.SaveKeyAssociationAsync(association);
Logger.LogDebug("COMPOSITE: Associazione creata con ID: {AssociationId} per record {RecordNumber}", associationId, recordNumber);
var associationId = await CredentialService.SaveKeyAssociationParallelAsync(association);
Logger.LogDebug("COMPOSITE: Associazione creata con ID: {AssociationId} per record {RecordNumber} (PARALLEL)", associationId, recordNumber);
}
catch (Exception ex)
{
@@ -2761,12 +2825,12 @@ public partial class DataCoupler : ComponentBase
var sourceKey = GenerateSourceKey(originalRecord);
if (string.IsNullOrEmpty(sourceKey)) return;
var existingAssociation = await CredentialService.FindKeyAssociationByValueAsync(
var existingAssociation = await CredentialService.FindKeyAssociationByValueParallelAsync(
sourceKey, selectedRestEntity?.Name ?? "", selectedRestCredential ?? "");
if (existingAssociation != null)
{
await CredentialService.DeleteKeyAssociationAsync(existingAssociation.Id);
await CredentialService.DeleteKeyAssociationParallelAsync(existingAssociation.Id);
Logger.LogInformation("COMPOSITE: Associazione non valida eliminata per record {RecordNumber}", recordNumber);
}
}
Data_Coupler/Pages/Index.razor
+2 -2
View File
@@ -1,4 +1,4 @@
@page "/"
@* @page "/"
<PageTitle>Index</PageTitle>
@@ -6,4 +6,4 @@
Welcome to your new app.
<SurveyPrompt Title="How is Blazor working for you?" />
<SurveyPrompt Title="How is Blazor working for you?" /> *@
Data_Coupler/Shared/NavMenu.razor
+7 -6
View File
@@ -10,6 +10,7 @@
<div class="@NavMenuCssClass nav-scrollable" @onclick="ToggleNavMenu">
<nav class="flex-column">
<div class="nav-item px-3">
@*
<NavLink class="nav-link" href="" Match="NavLinkMatch.All">
<span class="oi oi-home" aria-hidden="true"></span> Home
</NavLink>
@@ -23,20 +24,20 @@
<NavLink class="nav-link" href="fetchdata">
<span class="oi oi-list-rich" aria-hidden="true"></span> Fetch data
</NavLink>
</div> *@
@* <div class="nav-item"></div> *@
<NavLink class="nav-link" href="" Match="NavLinkMatch.All">
<span class="oi oi-transfer" aria-hidden="true"></span> Data Coupler
</NavLink>
</div>
<div class="nav-item px-3">
<NavLink class="nav-link" href="credentials">
<span class="oi oi-key" aria-hidden="true"></span> Gestione Credenziali
</NavLink>
</div>
<div class="nav-item px-3">
<NavLink class="nav-link" href="data-coupler">
<span class="oi oi-transfer" aria-hidden="true"></span> Data Coupler
</NavLink>
</div>
<div class="nav-item px-3">
<NavLink class="nav-link" href="key-associations">
<span class="oi oi-link-intact" aria-hidden="true"></span> Gestione Associazioni Chiave
<span class="oi oi-link-intact" aria-hidden="true"></span> Associazioni Chiavi
</NavLink>
</div>
<div class="nav-item px-3">
Data_Coupler/wwwroot/data/credentials.db-shm
BIN
View File
Binary file not shown.