diff --git a/Controllers/RecruitController.cs b/Controllers/RecruitController.cs index 8c1fbdd..a673934 100644 --- a/Controllers/RecruitController.cs +++ b/Controllers/RecruitController.cs @@ -717,9 +717,11 @@ namespace BMA.EHR.Recruit.Controllers var doc = await _minioService.UploadFileAsync(file); var import_doc_id = doc.Id.ToString("D"); - var fileContent = (await _minioService.DownloadFileAsync(doc.Id)).FileContent; - System.IO.File.WriteAllBytes(importFile, fileContent); - fileContent = null; + // Write file to disk directly from IFormFile instead of downloading back from MinIO + using (var stream = new FileStream(importFile, FileMode.Create)) + { + await file.CopyToAsync(stream); + } // สร้างรอบการบรรจุ var imported = new RecruitImport @@ -931,9 +933,11 @@ namespace BMA.EHR.Recruit.Controllers var doc = await _minioService.UploadFileAsync(file); var import_doc_id = doc.Id.ToString("D"); - var fileContent = (await _minioService.DownloadFileAsync(doc.Id)).FileContent; - System.IO.File.WriteAllBytes(importFile, fileContent); - fileContent = null; + // Write file to disk directly from IFormFile instead of downloading back from MinIO + using (var stream = new FileStream(importFile, FileMode.Create)) + { + await file.CopyToAsync(stream); + } // Enqueue background job var job = _importJobTracker.CreateJob(new ImportJobInfo diff --git a/Dockerfile b/Dockerfile index ea318e6..0ee996f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,4 +20,9 @@ RUN dotnet publish "BMA.EHR.Recruit.csproj" -c Release -o /app/publish /p:UseApp FROM base AS final WORKDIR /app COPY --from=publish /app/publish . + +# GC configuration for better memory management in containers +#ENV DOTNET_GCHeapHardLimit=1073741824 +#ENV DOTNET_GCConserveMemory=9 + ENTRYPOINT ["dotnet", "BMA.EHR.Recruit.dll"] \ No newline at end of file diff --git a/Services/ImportBackgroundService.cs b/Services/ImportBackgroundService.cs index f041390..cf33fd5 100644 --- a/Services/ImportBackgroundService.cs +++ b/Services/ImportBackgroundService.cs @@ -41,64 +41,61 @@ public class ImportBackgroundService : BackgroundService { var job = await _queue.DequeueAsync(stoppingToken); - _ = Task.Run(async () => - { - using var scope = _scopeFactory.CreateScope(); - var context = scope.ServiceProvider.GetRequiredService(); - var minioService = scope.ServiceProvider.GetRequiredService(); - var recruitService = scope.ServiceProvider.GetRequiredService(); - var notificationService = scope.ServiceProvider.GetRequiredService(); - var webHostEnv = scope.ServiceProvider.GetRequiredService(); - var logger = scope.ServiceProvider.GetRequiredService>(); + using var scope = _scopeFactory.CreateScope(); + var context = scope.ServiceProvider.GetRequiredService(); + var minioService = scope.ServiceProvider.GetRequiredService(); + var recruitService = scope.ServiceProvider.GetRequiredService(); + var notificationService = scope.ServiceProvider.GetRequiredService(); + var webHostEnv = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService>(); + try + { + _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running); + + switch (job.JobType) + { + case ImportJobType.CandidateFile: + await ProcessCandidateFileAsync(context, minioService, webHostEnv, job); + break; + case ImportJobType.CandidateFileById: + await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job); + break; + case ImportJobType.ScoreFile: + await ProcessScoreFileAsync(context, minioService, recruitService, job); + break; + case ImportJobType.ResultFile: + await ProcessResultFileAsync(context, recruitService, job); + break; + } + + _tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount); + + await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ"); + } + catch (Exception ex) + { + logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message); + _tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message); + + try { await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message); } catch { } + + // cleanup minio file on failure + if (!string.IsNullOrEmpty(job.ImportDocId)) + { + try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { } + } + } + finally + { + // cleanup temp file try { - _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running); - - switch (job.JobType) - { - case ImportJobType.CandidateFile: - await ProcessCandidateFileAsync(context, minioService, webHostEnv, job); - break; - case ImportJobType.CandidateFileById: - await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job); - break; - case ImportJobType.ScoreFile: - await ProcessScoreFileAsync(context, minioService, recruitService, job); - break; - case ImportJobType.ResultFile: - await ProcessResultFileAsync(context, recruitService, job); - break; - } - - _tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount); - - await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ"); + if (System.IO.File.Exists(job.ImportFile)) + System.IO.File.Delete(job.ImportFile); } - catch (Exception ex) - { - logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message); - _tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message); - - await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message); - - // cleanup minio file on failure - if (!string.IsNullOrEmpty(job.ImportDocId)) - { - try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { } - } - } - finally - { - // cleanup temp file - try - { - if (System.IO.File.Exists(job.ImportFile)) - System.IO.File.Delete(job.ImportFile); - } - catch { } - } - }, stoppingToken); + catch { } + } } } @@ -266,6 +263,8 @@ public class ImportBackgroundService : BackgroundService await _context.BulkInsertAsync(batchCertificates); await _context.BulkInsertAsync(batchEducations); + _context.ChangeTracker.Clear(); + batchRecruits.Clear(); batchAddresses.Clear(); batchPayments.Clear(); @@ -296,6 +295,8 @@ public class ImportBackgroundService : BackgroundService await _context.BulkInsertAsync(batchOccupations); await _context.BulkInsertAsync(batchCertificates); await _context.BulkInsertAsync(batchEducations); + + _context.ChangeTracker.Clear(); } } @@ -510,6 +511,8 @@ public class ImportBackgroundService : BackgroundService throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex); } + _context.ChangeTracker.Clear(); + // Clear all lists for next batch batchRecruits.Clear(); batchEducations.Clear(); @@ -537,6 +540,8 @@ public class ImportBackgroundService : BackgroundService var batchStartRow = row - batchCount + 1; throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex); } + + _context.ChangeTracker.Clear(); } } diff --git a/Services/ImportJobTracker.cs b/Services/ImportJobTracker.cs index ebd795b..d58a37c 100644 --- a/Services/ImportJobTracker.cs +++ b/Services/ImportJobTracker.cs @@ -45,9 +45,11 @@ public class ImportJobInfo public class ImportJobTracker { private readonly ConcurrentDictionary _jobs = new(); + private readonly TimeSpan _evictionAge = TimeSpan.FromHours(1); public ImportJobInfo CreateJob(ImportJobInfo job) { + EvictOldJobs(); _jobs[job.JobId] = job; return job; } @@ -66,7 +68,24 @@ public class ImportJobTracker if (errorMessage != null) job.ErrorMessage = errorMessage; if (status == ImportJobStatus.Completed || status == ImportJobStatus.Failed) + { job.CompletedAt = DateTime.Now; + // Clear request data to free memory for completed/failed jobs + job.Request = null; + job.Token = null; + } + } + } + + private void EvictOldJobs() + { + var cutoff = DateTime.Now - _evictionAge; + foreach (var kvp in _jobs) + { + if (kvp.Value.CompletedAt.HasValue && kvp.Value.CompletedAt.Value < cutoff) + { + _jobs.TryRemove(kvp.Key, out _); + } } } } diff --git a/Services/MinIOService.cs b/Services/MinIOService.cs index a56eba2..cf231b2 100644 --- a/Services/MinIOService.cs +++ b/Services/MinIOService.cs @@ -72,14 +72,13 @@ namespace BMA.EHR.Recruit.Services { var id = Guid.NewGuid(); file.CopyTo(ms); - var fileBytes = ms.ToArray(); - System.IO.MemoryStream filestream = new System.IO.MemoryStream(fileBytes); + ms.Position = 0; // Reset stream position for reading var request = new PutObjectRequest { BucketName = _bucketName, Key = id.ToString("D"), - InputStream = filestream, + InputStream = ms, ContentType = file.ContentType, CannedACL = S3CannedACL.PublicRead };