hrms-bangkok/hrms-api-recruit
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

fix OOM Error

Browse source
This commit is contained in:
Suphonchai Phoonsawat 2026-05-15 16:37:14 +07:00
parent 9c2caa3f4a
commit 8282adec25
5 changed files with 95 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Controllers/RecruitController.cs
View file

@ -717,9 +717,11 @@ namespace BMA.EHR.Recruit.Controllers
var doc = await _minioService.UploadFileAsync(file); var doc = await _minioService.UploadFileAsync(file);
var import_doc_id = doc.Id.ToString("D"); var import_doc_id = doc.Id.ToString("D");
var fileContent = (await _minioService.DownloadFileAsync(doc.Id)).FileContent; // Write file to disk directly from IFormFile instead of downloading back from MinIO
System.IO.File.WriteAllBytes(importFile, fileContent); using (var stream = new FileStream(importFile, FileMode.Create))
fileContent = null; {
await file.CopyToAsync(stream);
}
// สร้างรอบการบรรจุ // สร้างรอบการบรรจุ
var imported = new RecruitImport var imported = new RecruitImport
@ -931,9 +933,11 @@ namespace BMA.EHR.Recruit.Controllers
var doc = await _minioService.UploadFileAsync(file); var doc = await _minioService.UploadFileAsync(file);
var import_doc_id = doc.Id.ToString("D"); var import_doc_id = doc.Id.ToString("D");
var fileContent = (await _minioService.DownloadFileAsync(doc.Id)).FileContent; // Write file to disk directly from IFormFile instead of downloading back from MinIO
System.IO.File.WriteAllBytes(importFile, fileContent); using (var stream = new FileStream(importFile, FileMode.Create))
fileContent = null; {
await file.CopyToAsync(stream);
}
// Enqueue background job // Enqueue background job
var job = _importJobTracker.CreateJob(new ImportJobInfo var job = _importJobTracker.CreateJob(new ImportJobInfo

5
Dockerfile
View file

@ -20,4 +20,9 @@ RUN dotnet publish "BMA.EHR.Recruit.csproj" -c Release -o /app/publish /p:UseApp
FROM base AS final FROM base AS final
WORKDIR /app WORKDIR /app
COPY --from=publish /app/publish . COPY --from=publish /app/publish .
# GC configuration for better memory management in containers
#ENV DOTNET_GCHeapHardLimit=1073741824
#ENV DOTNET_GCConserveMemory=9
ENTRYPOINT ["dotnet", "BMA.EHR.Recruit.dll"] ENTRYPOINT ["dotnet", "BMA.EHR.Recruit.dll"]

113
Services/ImportBackgroundService.cs
View file

@ -41,64 +41,61 @@ public class ImportBackgroundService : BackgroundService
{ {
var job = await _queue.DequeueAsync(stoppingToken); var job = await _queue.DequeueAsync(stoppingToken);
_ = Task.Run(async () => using var scope = _scopeFactory.CreateScope();
{ var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
using var scope = _scopeFactory.CreateScope(); var minioService = scope.ServiceProvider.GetRequiredService<MinIOService>();
var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>(); var recruitService = scope.ServiceProvider.GetRequiredService<RecruitService>();
var minioService = scope.ServiceProvider.GetRequiredService<MinIOService>(); var notificationService = scope.ServiceProvider.GetRequiredService<NotificationService>();
var recruitService = scope.ServiceProvider.GetRequiredService<RecruitService>(); var webHostEnv = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>();
var notificationService = scope.ServiceProvider.GetRequiredService<NotificationService>(); var logger = scope.ServiceProvider.GetRequiredService<ILogger<ImportBackgroundService>>();
var webHostEnv = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<ImportBackgroundService>>();
try
{
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running);
switch (job.JobType)
{
case ImportJobType.CandidateFile:
await ProcessCandidateFileAsync(context, minioService, webHostEnv, job);
break;
case ImportJobType.CandidateFileById:
await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job);
break;
case ImportJobType.ScoreFile:
await ProcessScoreFileAsync(context, minioService, recruitService, job);
break;
case ImportJobType.ResultFile:
await ProcessResultFileAsync(context, recruitService, job);
break;
}
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount);
await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ");
}
catch (Exception ex)
{
logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message);
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message);
try { await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message); } catch { }
// cleanup minio file on failure
if (!string.IsNullOrEmpty(job.ImportDocId))
{
try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { }
}
}
finally
{
// cleanup temp file
try try
{ {
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running); if (System.IO.File.Exists(job.ImportFile))
System.IO.File.Delete(job.ImportFile);
switch (job.JobType)
{
case ImportJobType.CandidateFile:
await ProcessCandidateFileAsync(context, minioService, webHostEnv, job);
break;
case ImportJobType.CandidateFileById:
await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job);
break;
case ImportJobType.ScoreFile:
await ProcessScoreFileAsync(context, minioService, recruitService, job);
break;
case ImportJobType.ResultFile:
await ProcessResultFileAsync(context, recruitService, job);
break;
}
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount);
await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ");
} }
catch (Exception ex) catch { }
{ }
logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message);
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message);
await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message);
// cleanup minio file on failure
if (!string.IsNullOrEmpty(job.ImportDocId))
{
try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { }
}
}
finally
{
// cleanup temp file
try
{
if (System.IO.File.Exists(job.ImportFile))
System.IO.File.Delete(job.ImportFile);
}
catch { }
}
}, stoppingToken);
} }
} }
@ -266,6 +263,8 @@ public class ImportBackgroundService : BackgroundService
await _context.BulkInsertAsync(batchCertificates); await _context.BulkInsertAsync(batchCertificates);
await _context.BulkInsertAsync(batchEducations); await _context.BulkInsertAsync(batchEducations);
_context.ChangeTracker.Clear();
batchRecruits.Clear(); batchRecruits.Clear();
batchAddresses.Clear(); batchAddresses.Clear();
batchPayments.Clear(); batchPayments.Clear();
@ -296,6 +295,8 @@ public class ImportBackgroundService : BackgroundService
await _context.BulkInsertAsync(batchOccupations); await _context.BulkInsertAsync(batchOccupations);
await _context.BulkInsertAsync(batchCertificates); await _context.BulkInsertAsync(batchCertificates);
await _context.BulkInsertAsync(batchEducations); await _context.BulkInsertAsync(batchEducations);
_context.ChangeTracker.Clear();
} }
} }
@ -510,6 +511,8 @@ public class ImportBackgroundService : BackgroundService
throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex); throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
} }
_context.ChangeTracker.Clear();
// Clear all lists for next batch // Clear all lists for next batch
batchRecruits.Clear(); batchRecruits.Clear();
batchEducations.Clear(); batchEducations.Clear();
@ -537,6 +540,8 @@ public class ImportBackgroundService : BackgroundService
var batchStartRow = row - batchCount + 1; var batchStartRow = row - batchCount + 1;
throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex); throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
} }
_context.ChangeTracker.Clear();
} }
} }

19
Services/ImportJobTracker.cs
View file

@ -45,9 +45,11 @@ public class ImportJobInfo
public class ImportJobTracker public class ImportJobTracker
{ {
private readonly ConcurrentDictionary<string, ImportJobInfo> _jobs = new(); private readonly ConcurrentDictionary<string, ImportJobInfo> _jobs = new();
private readonly TimeSpan _evictionAge = TimeSpan.FromHours(1);
public ImportJobInfo CreateJob(ImportJobInfo job) public ImportJobInfo CreateJob(ImportJobInfo job)
{ {
EvictOldJobs();
_jobs[job.JobId] = job; _jobs[job.JobId] = job;
return job; return job;
} }
@ -66,7 +68,24 @@ public class ImportJobTracker
if (errorMessage != null) if (errorMessage != null)
job.ErrorMessage = errorMessage; job.ErrorMessage = errorMessage;
if (status == ImportJobStatus.Completed || status == ImportJobStatus.Failed) if (status == ImportJobStatus.Completed || status == ImportJobStatus.Failed)
{
job.CompletedAt = DateTime.Now; job.CompletedAt = DateTime.Now;
// Clear request data to free memory for completed/failed jobs
job.Request = null;
job.Token = null;
}
}
}
private void EvictOldJobs()
{
var cutoff = DateTime.Now - _evictionAge;
foreach (var kvp in _jobs)
{
if (kvp.Value.CompletedAt.HasValue && kvp.Value.CompletedAt.Value < cutoff)
{
_jobs.TryRemove(kvp.Key, out _);
}
} }
} }
} }

5
Services/MinIOService.cs
View file

@ -72,14 +72,13 @@ namespace BMA.EHR.Recruit.Services
{ {
var id = Guid.NewGuid(); var id = Guid.NewGuid();
file.CopyTo(ms); file.CopyTo(ms);
var fileBytes = ms.ToArray(); ms.Position = 0; // Reset stream position for reading
System.IO.MemoryStream filestream = new System.IO.MemoryStream(fileBytes);
var request = new PutObjectRequest var request = new PutObjectRequest
{ {
BucketName = _bucketName, BucketName = _bucketName,
Key = id.ToString("D"), Key = id.ToString("D"),
InputStream = filestream, InputStream = ms,
ContentType = file.ContentType, ContentType = file.ContentType,
CannedACL = S3CannedACL.PublicRead CannedACL = S3CannedACL.PublicRead
}; };