hrms-api-recruit/Services/ImportBackgroundService.cs
Suphonchai Phoonsawat 13a8e309e7
All checks were successful
Build & Deploy on Dev / build (push) Successful in 49s
fix minio file
2026-05-19 16:57:33 +07:00

962 lines
46 KiB
C#

using System.Threading.Channels;
using BMA.EHR.Recruit.Core;
using BMA.EHR.Recruit.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using System.Net.Http.Headers;
using BMA.EHR.Recruit.Data;
using BMA.EHR.Recruit.Extensions;
using EFCore.BulkExtensions;
using BMA.EHR.Recruit.Models.Recruits;
using BMA.EHR.Recruit.Requests.Recruits;
using ExcelDataReader;
namespace BMA.EHR.Recruit.Services;
public class ImportBackgroundService : BackgroundService
{
private readonly ImportJobQueue _queue;
private readonly ImportJobTracker _tracker;
private readonly IServiceScopeFactory _scopeFactory;
private readonly ILogger<ImportBackgroundService> _logger;
public ImportBackgroundService(
ImportJobQueue queue,
ImportJobTracker tracker,
IServiceScopeFactory scopeFactory,
ILogger<ImportBackgroundService> logger)
{
_queue = queue;
_tracker = tracker;
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("ImportBackgroundService started.");
while (!stoppingToken.IsCancellationRequested)
{
var job = await _queue.DequeueAsync(stoppingToken);
using var scope = _scopeFactory.CreateScope();
var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
var minioService = scope.ServiceProvider.GetRequiredService<MinIOService>();
var recruitService = scope.ServiceProvider.GetRequiredService<RecruitService>();
var notificationService = scope.ServiceProvider.GetRequiredService<NotificationService>();
var webHostEnv = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>();
var logger = scope.ServiceProvider.GetRequiredService<ILogger<ImportBackgroundService>>();
try
{
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running);
switch (job.JobType)
{
case ImportJobType.CandidateFile:
await ProcessCandidateFileAsync(context, minioService, webHostEnv, job);
break;
case ImportJobType.CandidateFileById:
await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job);
break;
case ImportJobType.ScoreFile:
await ProcessScoreFileAsync(context, minioService, recruitService, job);
break;
case ImportJobType.ResultFile:
await ProcessResultFileAsync(context, recruitService, job);
break;
}
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount);
await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ");
job.Token = null; // Clear token after notification sent
}
catch (Exception ex)
{
logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message);
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message);
try { await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message); } catch { }
job.Token = null; // Clear token after notification sent
// cleanup minio file on failure
if (!string.IsNullOrEmpty(job.ImportDocId))
{
try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { }
}
}
finally
{
// cleanup temp file
try
{
if (System.IO.File.Exists(job.ImportFile))
System.IO.File.Delete(job.ImportFile);
}
catch { }
}
}
}
#region CandidateFile
private async Task ProcessCandidateFileAsync(ApplicationDbContext _context, MinIOService _minioService, IWebHostEnvironment _webHostEnv, ImportJobInfo job)
{
var imported = await _context.RecruitImports.FindAsync(job.RecruitImportId);
if (imported == null) throw new Exception("RecruitImport not found");
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
using var stream = System.IO.File.OpenRead(job.ImportFile);
using var reader = ExcelReaderFactory.CreateReader(stream);
do
{
// Read header row (row 1) to build column index map
if (!reader.Read()) continue;
var cols = new string[reader.FieldCount];
for (int c = 0; c < reader.FieldCount; c++)
cols[c] = reader.GetValue(c)?.ToString() ?? "";
int batchCount = 0;
const int batchSize = 500;
int totalProcessed = 0;
var batchRecruits = new List<Models.Recruits.Recruit>();
var batchEducations = new List<RecruitEducation>();
var batchOccupations = new List<RecruitOccupation>();
var batchAddresses = new List<RecruitAddress>();
var batchPayments = new List<RecruitPayment>();
var batchCertificates = new List<RecruitCertificate>();
while (reader.Read())
{
var cell1 = reader.GetValue(0)?.ToString();
if (string.IsNullOrEmpty(cell1)) break;
var r = new Models.Recruits.Recruit();
r.ExamId = GetCellValue(reader, cols, CandidateFileHeader.ExamID);
r.CitizenId = GetCellValue(reader, cols, CandidateFileHeader.PersonalID);
r.Prefix = GetCellValue(reader, cols, CandidateFileHeader.Prefix);
r.FirstName = GetCellValue(reader, cols, CandidateFileHeader.FirstName);
r.LastName = GetCellValue(reader, cols, CandidateFileHeader.LastName);
r.Gendor = GetCellValue(reader, cols, CandidateFileHeader.Gender);
r.National = GetCellValue(reader, cols, CandidateFileHeader.National).IsNull("");
r.Race = GetCellValue(reader, cols, CandidateFileHeader.Race).IsNull("");
r.Religion = GetCellValue(reader, cols, CandidateFileHeader.Religion).IsNull("");
r.DateOfBirth = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.DateOfBirth).ToDateTime(DateTimeFormat.Ymd, "-"));
r.Marry = GetCellValue(reader, cols, CandidateFileHeader.Marry);
r.Isspecial = "N";
r.CitizenCardIssuer = GetCellValue(reader, cols, CandidateFileHeader.PersonalCardIssue);
r.CitizenCardExpireDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.PersonalCardExpireDate).ToDateTime(DateTimeFormat.Ymd, "-"));
r.ApplyDate = GetCellDateTime(reader, cols, CandidateFileHeader.ApplyDate) ?? DateTime.MinValue;
r.PositionName = GetCellValue(reader, cols, CandidateFileHeader.PositionName).IsNull("");
r.PositionType = GetCellValue(reader, cols, CandidateFileHeader.PositionType).IsNull("");
r.PositionLevel = GetCellValue(reader, cols, CandidateFileHeader.PositionLevel).IsNull("");
// address
var address = new RecruitAddress()
{
Address = GetCellValue(reader, cols, CandidateFileHeader.Address),
Moo = GetCellValue(reader, cols, CandidateFileHeader.Moo),
Soi = GetCellValue(reader, cols, CandidateFileHeader.Soi),
Road = GetCellValue(reader, cols, CandidateFileHeader.Road),
District = GetCellValue(reader, cols, CandidateFileHeader.District),
Amphur = GetCellValue(reader, cols, CandidateFileHeader.Amphur),
Province = GetCellValue(reader, cols, CandidateFileHeader.Province),
ZipCode = GetCellValue(reader, cols, CandidateFileHeader.ZipCode),
Telephone = GetCellValue(reader, cols, CandidateFileHeader.Telephone),
Mobile = GetCellValue(reader, cols, CandidateFileHeader.Mobile),
Address1 = GetCellValue(reader, cols, CandidateFileHeader.Address1),
Moo1 = GetCellValue(reader, cols, CandidateFileHeader.Moo1),
Soi1 = GetCellValue(reader, cols, CandidateFileHeader.Soi1),
Road1 = GetCellValue(reader, cols, CandidateFileHeader.Road1),
District1 = GetCellValue(reader, cols, CandidateFileHeader.District1),
Amphur1 = GetCellValue(reader, cols, CandidateFileHeader.Amphur1),
Province1 = GetCellValue(reader, cols, CandidateFileHeader.Province1),
ZipCode1 = GetCellValue(reader, cols, CandidateFileHeader.ZipCode1),
};
// payment
var payment = new RecruitPayment()
{
PaymentId = GetCellValue(reader, cols, CandidateFileHeader.PaymentID),
CompanyCode = GetCellValue(reader, cols, CandidateFileHeader.CompanyCode),
TextFile = GetCellValue(reader, cols, CandidateFileHeader.TextFile),
BankCode = GetCellValue(reader, cols, CandidateFileHeader.BankCode),
AccountNumber = GetCellValue(reader, cols, CandidateFileHeader.AccouontNumer),
TransDate = GetCellValue(reader, cols, CandidateFileHeader.TransDate),
TransTime = GetCellValue(reader, cols, CandidateFileHeader.TransTime),
CustomerName = GetCellValue(reader, cols, CandidateFileHeader.CustomerName),
RefNo1 = GetCellValue(reader, cols, CandidateFileHeader.RefNo1),
TermBranch = GetCellValue(reader, cols, CandidateFileHeader.TermBranch),
TellerId = GetCellValue(reader, cols, CandidateFileHeader.TellerID),
CreditDebit = GetCellValue(reader, cols, CandidateFileHeader.CreditDebit),
PaymentType = GetCellValue(reader, cols, CandidateFileHeader.Type),
ChequeNo = GetCellValue(reader, cols, CandidateFileHeader.ChequeNo),
Amount = GetCellDecimal(reader, cols, CandidateFileHeader.Amount),
ChqueBankCode = GetCellValue(reader, cols, CandidateFileHeader.ChqBankCode)
};
// occupation
var occupation = new RecruitOccupation()
{
Occupation = GetCellValue(reader, cols, CandidateFileHeader.Occupation),
Position = GetCellValue(reader, cols, CandidateFileHeader.Position),
Workplace = GetCellValue(reader, cols, CandidateFileHeader.Workplace),
Telephone = GetCellValue(reader, cols, CandidateFileHeader.WorkplaceTelephone),
WorkAge = GetCellValue(reader, cols, CandidateFileHeader.WorkAge),
};
// certificate
var certificate = new RecruitCertificate()
{
CertificateNo = GetCellValue(reader, cols, CandidateFileHeader.CertificateNo),
Description = GetCellValue(reader, cols, CandidateFileHeader.CertificateDesc),
IssueDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.CertificateIssueDate).ToDateTime(DateTimeFormat.Ymd, "-")),
ExpiredDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.CertificateExpireDate).ToDateTime(DateTimeFormat.Ymd, "-"))
};
var education = new RecruitEducation()
{
Degree = GetCellValue(reader, cols, CandidateFileHeader.Degree),
Major = GetCellValue(reader, cols, CandidateFileHeader.Major),
MajorGroupId = GetCellValue(reader, cols, CandidateFileHeader.MajorGroupID),
MajorGroupName = GetCellValue(reader, cols, CandidateFileHeader.MajorGroupName),
University = GetCellValue(reader, cols, CandidateFileHeader.University),
GPA = GetCellDouble(reader, cols, CandidateFileHeader.GPA),
Specialist = GetCellValue(reader, cols, CandidateFileHeader.SpecialList),
HighDegree = GetCellValue(reader, cols, CandidateFileHeader.HighDegree),
BachelorDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.BachelorDate).ToDateTime(DateTimeFormat.Ymd, "-"))
};
r.Addresses.Add(address);
r.Payments.Add(payment);
r.Occupations.Add(occupation);
r.Certificates.Add(certificate);
r.Educations.Add(education);
r.RecruitImport = imported;
batchRecruits.Add(r);
batchAddresses.Add(address);
batchPayments.Add(payment);
batchOccupations.Add(occupation);
batchCertificates.Add(certificate);
batchEducations.Add(education);
batchCount++;
totalProcessed++;
if (batchCount >= batchSize)
{
await _context.BulkInsertAsync(batchRecruits, options => { options.SetOutputIdentity = true; });
for (int j = 0; j < batchRecruits.Count; j++)
{
batchAddresses[j].Recruit = batchRecruits[j];
batchPayments[j].Recruit = batchRecruits[j];
batchOccupations[j].Recruit = batchRecruits[j];
batchCertificates[j].Recruit = batchRecruits[j];
batchEducations[j].Recruit = batchRecruits[j];
}
await _context.BulkInsertAsync(batchAddresses);
await _context.BulkInsertAsync(batchPayments);
await _context.BulkInsertAsync(batchOccupations);
await _context.BulkInsertAsync(batchCertificates);
await _context.BulkInsertAsync(batchEducations);
_context.ChangeTracker.Clear();
batchRecruits.Clear();
batchAddresses.Clear();
batchPayments.Clear();
batchOccupations.Clear();
batchCertificates.Clear();
batchEducations.Clear();
batchCount = 0;
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
}
}
// Process remaining records
if (batchRecruits.Count > 0)
{
await _context.BulkInsertAsync(batchRecruits, options => { options.SetOutputIdentity = true; });
for (int j = 0; j < batchRecruits.Count; j++)
{
batchAddresses[j].Recruit = batchRecruits[j];
batchPayments[j].Recruit = batchRecruits[j];
batchOccupations[j].Recruit = batchRecruits[j];
batchCertificates[j].Recruit = batchRecruits[j];
batchEducations[j].Recruit = batchRecruits[j];
}
await _context.BulkInsertAsync(batchAddresses);
await _context.BulkInsertAsync(batchPayments);
await _context.BulkInsertAsync(batchOccupations);
await _context.BulkInsertAsync(batchCertificates);
await _context.BulkInsertAsync(batchEducations);
_context.ChangeTracker.Clear();
}
} while (reader.NextResult());
job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
}
#endregion
#region CandidateFileById
private async Task ProcessCandidateFileByIdAsync(ApplicationDbContext _context, MinIOService _minioService, RecruitService _recruitService, IWebHostEnvironment _webHostEnv, ImportJobInfo job)
{
var imported = await _context.RecruitImports.FindAsync(job.RecruitImportId);
if (imported == null) throw new Exception("RecruitImport not found");
// Save import history using regular SaveChanges (small operation)
imported.ImportHostories.Add(new RecruitImportHistory
{
Description = "นำเข้าข้อมูลผู้สมัครสอบแข่งขัน",
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator",
});
await _context.SaveChangesAsync();
_context.ChangeTracker.Clear();
var importId = imported.Id;
var importRef = _context.Attach(new RecruitImport { Id = importId }).Entity;
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
using var stream = System.IO.File.OpenRead(job.ImportFile);
using var reader = ExcelReaderFactory.CreateReader(stream);
do
{
// Skip header row
if (!reader.Read()) continue;
int row = 2;
int batchCount = 0;
const int batchSize = 500;
int totalProcessed = 0;
var batchRecruits = new List<Models.Recruits.Recruit>();
var batchEducations = new List<RecruitEducation>();
var batchOccupations = new List<RecruitOccupation>();
var batchAddresses = new List<RecruitAddress>();
var batchPayments = new List<RecruitPayment>();
while (reader.Read())
{
var cell1 = reader.GetValue(0)?.ToString();
if (string.IsNullOrEmpty(cell1)) break;
try
{
var r = new Models.Recruits.Recruit();
r.Id = Guid.NewGuid();
r.ExamId = reader.GetValue(0)?.ToString() ?? "";
r.PositionName = reader.GetValue(2)?.ToString() ?? "";
r.HddPosition = reader.GetValue(3)?.ToString() ?? "";
r.Prefix = reader.GetValue(4)?.ToString() == "อื่น ๆ" ? reader.GetValue(5)?.ToString() ?? "" : reader.GetValue(4)?.ToString() ?? "";
r.FirstName = reader.GetValue(6)?.ToString() ?? "";
r.LastName = reader.GetValue(7)?.ToString() ?? "";
r.Gendor = reader.GetValue(97)?.ToString() ?? "";
r.National = reader.GetValue(8)?.ToString() ?? "";
r.Race = "";
r.Religion = reader.GetValue(9)?.ToString() ?? "";
r.DateOfBirth = !string.IsNullOrWhiteSpace(reader.GetValue(10)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(10)?.ToString() ?? "", "dd/MM/yyyy") : null;
r.CitizenId = reader.GetValue(11)?.ToString() ?? "";
r.typeTest = reader.GetValue(12)?.ToString() ?? "";
r.Marry = "";
r.Isspecial = "N";
r.CitizenCardExpireDate = null;
r.ModifiedDate = null;
r.ApplyDate = !string.IsNullOrWhiteSpace(reader.GetValue(86)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(86)?.ToString() ?? "", "dd/MM/yyyy") : null;
r.PositionType = "";
r.PositionLevel = "";
r.CreatedAt = DateTime.Now;
r.CreatedUserId = job.UserId ?? "";
r.CreatedFullName = job.FullName ?? "System Administrator";
r.LastUpdatedAt = DateTime.Now;
r.LastUpdateUserId = job.UserId ?? "";
r.LastUpdateFullName = job.FullName ?? "System Administrator";
r.RecruitImport = importRef;
// Store child entities in separate lists for bulk insert
var education = new RecruitEducation()
{
Id = Guid.NewGuid(),
Degree = reader.GetValue(17)?.ToString() ?? "",
Major = reader.GetValue(18)?.ToString() == "อื่น ๆ" ? reader.GetValue(19)?.ToString() ?? "" : reader.GetValue(18)?.ToString() ?? "",
MajorGroupId = "",
MajorGroupName = "",
University = reader.GetValue(20)?.ToString() == "อื่น ๆ" ? reader.GetValue(21)?.ToString() ?? "" : reader.GetValue(20)?.ToString() ?? "",
GPA = GetReaderDouble(reader, 25),
Specialist = "",
HighDegree = reader.GetValue(26)?.ToString() ?? "",
BachelorDate = !string.IsNullOrWhiteSpace(reader.GetValue(24)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(24)?.ToString() ?? "", "dd/MM/yyyy") : null,
Recruit = r,
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator"
};
var occupation = new RecruitOccupation()
{
Id = Guid.NewGuid(),
Occupation = reader.GetValue(32)?.ToString() == "อื่น ๆ" ? reader.GetValue(33)?.ToString() ?? "" : reader.GetValue(32)?.ToString() ?? "",
Position = reader.GetValue(36)?.ToString() ?? "",
Workplace = $"{(reader.GetValue(35)?.ToString() ?? "")} {(reader.GetValue(34)?.ToString() ?? "")}",
Telephone = "",
WorkAge = "",
Recruit = r,
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator"
};
var address = new RecruitAddress()
{
Id = Guid.NewGuid(),
Address = $"{(reader.GetValue(48)?.ToString() ?? "")} {(reader.GetValue(49)?.ToString() ?? "")}",
Moo = reader.GetValue(50)?.ToString() ?? "",
Soi = reader.GetValue(51)?.ToString() ?? "",
Road = reader.GetValue(52)?.ToString() ?? "",
District = reader.GetValue(53)?.ToString() ?? "",
Amphur = reader.GetValue(54)?.ToString() ?? "",
Province = reader.GetValue(55)?.ToString() ?? "",
ZipCode = (reader.GetValue(56)?.ToString() ?? "").Trim(),
Telephone = reader.GetValue(57)?.ToString() ?? "",
Mobile = "",
Address1 = $"{(reader.GetValue(60)?.ToString() ?? "")} {(reader.GetValue(61)?.ToString() ?? "")}",
Moo1 = reader.GetValue(62)?.ToString() ?? "",
Soi1 = reader.GetValue(63)?.ToString() ?? "",
Road1 = reader.GetValue(64)?.ToString() ?? "",
District1 = reader.GetValue(65)?.ToString() ?? "",
Amphur1 = reader.GetValue(66)?.ToString() ?? "",
Province1 = reader.GetValue(67)?.ToString() ?? "",
ZipCode1 = (reader.GetValue(68)?.ToString() ?? "").Trim(),
Recruit = r,
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator"
};
var payment = new RecruitPayment()
{
Id = Guid.NewGuid(),
PaymentId = reader.GetValue(103)?.ToString() ?? "",
CompanyCode = reader.GetValue(104)?.ToString() ?? "",
TextFile = reader.GetValue(105)?.ToString() ?? "",
BankCode = reader.GetValue(106)?.ToString() ?? "",
AccountNumber = reader.GetValue(107)?.ToString() ?? "",
TransDate = reader.GetValue(108)?.ToString() ?? "",
TransTime = reader.GetValue(109)?.ToString() ?? "",
CustomerName = reader.GetValue(110)?.ToString() ?? "",
RefNo1 = reader.GetValue(111)?.ToString() ?? "",
TermBranch = reader.GetValue(112)?.ToString() ?? "",
TellerId = reader.GetValue(113)?.ToString() ?? "",
CreditDebit = reader.GetValue(114)?.ToString() ?? "",
PaymentType = reader.GetValue(115)?.ToString() ?? "",
ChequeNo = reader.GetValue(116)?.ToString() ?? "",
Amount = GetReaderDecimal(reader, 117),
ChqueBankCode = reader.GetValue(118)?.ToString() ?? "",
Recruit = r,
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator"
};
batchRecruits.Add(r);
batchEducations.Add(education);
batchOccupations.Add(occupation);
batchAddresses.Add(address);
batchPayments.Add(payment);
}
catch (Exception ex)
{
throw new Exception($"Row {row}: {ex.Message}", ex);
}
row++;
batchCount++;
totalProcessed++;
if (batchCount >= batchSize)
{
try
{
await _context.BulkInsertAsync(batchRecruits);
await _context.BulkInsertAsync(batchEducations);
await _context.BulkInsertAsync(batchOccupations);
await _context.BulkInsertAsync(batchAddresses);
await _context.BulkInsertAsync(batchPayments);
}
catch (Exception ex)
{
var batchStartRow = row - batchCount + 1;
throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
}
_context.ChangeTracker.Clear();
// Clear all lists for next batch
batchRecruits.Clear();
batchEducations.Clear();
batchOccupations.Clear();
batchAddresses.Clear();
batchPayments.Clear();
batchCount = 0;
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
}
}
// Process remaining records
if (batchRecruits.Count > 0)
{
try
{
await _context.BulkInsertAsync(batchRecruits);
await _context.BulkInsertAsync(batchEducations);
await _context.BulkInsertAsync(batchOccupations);
await _context.BulkInsertAsync(batchAddresses);
await _context.BulkInsertAsync(batchPayments);
}
catch (Exception ex)
{
var batchStartRow = row - batchCount + 1;
throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
}
_context.ChangeTracker.Clear();
}
} while (reader.NextResult());
job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
}
#endregion
#region ScoreFile
private async Task ProcessScoreFileAsync(ApplicationDbContext _context, MinIOService _minioService, RecruitService _recruitService, ImportJobInfo job)
{
var rec_import = await _context.RecruitImports.AsQueryable()
.Include(x => x.ScoreImport)
.ThenInclude(x => x.Scores)
.Include(x => x.ImportHostories)
.FirstOrDefaultAsync(x => x.Id == job.RecruitImportId);
if (rec_import == null) throw new Exception("RecruitImport not found");
var rec_import_id = rec_import.Id;
var rec_import_year = rec_import.Year;
var hasExistingScores = rec_import.ScoreImport != null && rec_import.ScoreImport.Scores != null;
if (hasExistingScores)
{
var existingScores = rec_import.ScoreImport.Scores.ToList();
await _context.BulkDeleteAsync(existingScores);
}
// Clear tracker to avoid stale references after BulkDelete (which bypasses EF tracking)
_context.ChangeTracker.Clear();
// Add history record — set FK shadow property directly to avoid Attach side-effects
var historyEntry = _context.RecruitImportHistories.Add(new RecruitImportHistory
{
Description = "นำเข้าข้อมูลผลคะแนนสอบ",
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator",
});
historyEntry.Property("RecruitImportId").CurrentValue = rec_import_id;
// get doc from minio (MinIOService already saves Document to its own context)
var doc = await _minioService.UploadFileAsync(new DummyFormFile(job.ImportFile));
var imported = new ScoreImport();
imported.Year = rec_import_year;
imported.RecruitImportId = rec_import_id;
imported.CreatedAt = DateTime.Now;
imported.CreatedUserId = job.UserId ?? "";
imported.CreatedFullName = job.FullName ?? "System Administrator";
imported.LastUpdatedAt = DateTime.Now;
imported.LastUpdateUserId = job.UserId ?? "";
imported.LastUpdateFullName = job.FullName ?? "System Administrator";
imported.Scores = new List<RecruitScore>();
// Save ScoreImport — use shadow FK for ImportFile to avoid re-inserting Document
var scoreEntry = _context.ScoreImports.Add(imported);
scoreEntry.Property("ImportFileId").CurrentValue = doc.Id;
await _context.SaveChangesAsync();
_context.ChangeTracker.Clear();
// preload recruits using AsNoTracking to avoid EF tracking overhead
var recruitsDict = await _context.Recruits
.AsNoTracking()
.Where(x => x.RecruitImport.Id == rec_import.Id)
.GroupBy(x => x.ExamId)
.Where(g => g.Count() == 1)
.Select(g => g.First())
.ToDictionaryAsync(x => x.ExamId, x => x);
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
using var stream = System.IO.File.OpenRead(job.ImportFile);
using var reader = ExcelReaderFactory.CreateReader(stream);
do
{
// Read header rows (rows 1-7), then data starts at row 8
// Skip 7 rows: first 7 are header/metadata
for (int skip = 0; skip < 7; skip++)
{
if (!reader.Read()) break;
}
var cols = new string[reader.FieldCount];
// Use current row (row 7) as header reference — not actually used for ScoreFile column mapping
// ScoreFile uses hardcoded column indices
int batchCount = 0;
const int batchSize = 500;
int totalProcessed = 0;
var batchScores = new List<RecruitScore>();
while (reader.Read())
{
var cell1 = reader.GetValue(0)?.ToString();
if (string.IsNullOrEmpty(cell1)) break;
var r = new RecruitScore();
r.ExamId = reader.GetValue(1)?.ToString();
if (!string.IsNullOrEmpty(r.ExamId) && recruitsDict.TryGetValue(r.ExamId, out var recruit))
{
r.CitizenId = reader.GetValue(2)?.ToString()?.Trim();
r.FullA = 200;
r.SumA = string.IsNullOrWhiteSpace(reader.GetValue(4)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(4)), 2);
r.PercentageA = string.IsNullOrWhiteSpace(reader.GetValue(5)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(5)), 2);
r.AStatus = string.IsNullOrWhiteSpace(reader.GetValue(6)?.ToString()) ? "" : reader.GetValue(6)?.ToString();
r.SumAB = string.IsNullOrWhiteSpace(reader.GetValue(4)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(4)), 2);
r.ABStatus = string.IsNullOrWhiteSpace(reader.GetValue(6)?.ToString()) ? "" : reader.GetValue(6)?.ToString();
r.FullC = 50;
r.SumC = string.IsNullOrWhiteSpace(reader.GetValue(7)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(7)), 2);
r.FullD = 50;
r.SumD = string.IsNullOrWhiteSpace(reader.GetValue(8)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(8)), 2);
r.SumCD = string.IsNullOrWhiteSpace(reader.GetValue(9)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(9)), 2);
r.PercentageC = string.IsNullOrWhiteSpace(reader.GetValue(10)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(10)), 2);
r.CStatus = string.IsNullOrWhiteSpace(reader.GetValue(11)?.ToString()) ? "" : reader.GetValue(11)?.ToString();
r.FullScore = 300;
r.TotalScore = string.IsNullOrWhiteSpace(reader.GetValue(12)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(12)), 2);
var examStatusCol7 = reader.GetValue(6)?.ToString()?.Trim();
var examStatusCol14 = reader.GetValue(13)?.ToString()?.Trim();
r.ExamStatus =
examStatusCol7 == "ขาดสอบ" ? "ขส." :
examStatusCol14 == "ได้" ? "ผ่าน" :
examStatusCol14 == "ตก" ? "ไม่ผ่าน" : "-";
r.RemarkScore = string.IsNullOrWhiteSpace(reader.GetValue(14)?.ToString()) ? string.Empty : reader.GetValue(14)?.ToString();
var examAttr = reader.GetValue(15)?.ToString()?.Trim();
r.ExamAttribute =
examAttr == "ผ่าน" ? "มีคุณสมบัติ" :
examAttr == "ไม่ผ่าน" ? "ไม่มีคุณสมบัติ" : "";
r.Major = reader.Name; // worksheet name
r.CreatedAt = DateTime.Now;
r.CreatedUserId = job.UserId ?? "";
r.CreatedFullName = job.FullName ?? "System Administrator";
r.LastUpdatedAt = DateTime.Now;
r.LastUpdateUserId = job.UserId ?? "";
r.LastUpdateFullName = job.FullName ?? "System Administrator";
r.ScoreImport = imported;
batchScores.Add(r);
}
batchCount++;
totalProcessed++;
if (batchCount >= batchSize)
{
await _context.BulkInsertAsync(batchScores);
batchScores.Clear();
batchCount = 0;
_tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
}
}
// Process remaining records
if (batchScores.Count > 0)
{
await _context.BulkInsertAsync(batchScores);
}
} while (reader.NextResult());
job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
}
#endregion
#region ResultFile
private async Task ProcessResultFileAsync(ApplicationDbContext _context, RecruitService _recruitService, ImportJobInfo job)
{
var rec_import = await _context.RecruitImports.AsQueryable()
.Include(x => x.ScoreImport)
.ThenInclude(x => x.Scores)
.Include(x => x.ImportHostories)
.FirstOrDefaultAsync(x => x.Id == job.RecruitImportId);
if (rec_import == null) throw new Exception("RecruitImport not found");
// update old scores
if (rec_import.ScoreImport != null && rec_import.ScoreImport.Scores != null)
{
var oldScores = rec_import.ScoreImport.Scores
.Where(x => !string.IsNullOrEmpty(x.Number))
.ToList();
if (oldScores.Count > 0)
{
foreach (var x in oldScores)
{
x.Number = string.Empty;
x.RemarkExamOrder = string.Empty;
}
await _context.BulkUpdateAsync(oldScores);
}
}
rec_import.ImportHostories.Add(new RecruitImportHistory
{
Description = "นำเข้าข้อมูลผลการสอบ",
CreatedAt = DateTime.Now,
CreatedUserId = job.UserId ?? "",
CreatedFullName = job.FullName ?? "System Administrator",
LastUpdatedAt = DateTime.Now,
LastUpdateUserId = job.UserId ?? "",
LastUpdateFullName = job.FullName ?? "System Administrator",
});
await _context.SaveChangesAsync();
_context.ChangeTracker.Clear();
// preload scores using AsNoTracking to avoid EF tracking overhead
var scoreList = await _context.RecruitScores
.AsNoTracking()
.Where(s => s.ScoreImport.RecruitImportId == rec_import.Id && !string.IsNullOrEmpty(s.ExamId))
.GroupBy(x => x.ExamId)
.Where(g => g.Count() == 1)
.Select(g => g.First())
.ToListAsync();
var score = scoreList.ToDictionary(s => s.ExamId!, s => s);
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
using var stream = System.IO.File.OpenRead(job.ImportFile);
using var reader = ExcelReaderFactory.CreateReader(stream);
do
{
// Skip 6 header rows, data starts at row 7
for (int skip = 0; skip < 6; skip++)
{
if (!reader.Read()) break;
}
int batchCount = 0;
const int batchSize = 500;
var batchUpdates = new List<RecruitScore>();
while (reader.Read())
{
var examId = reader.GetValue(1)?.ToString();
if (string.IsNullOrWhiteSpace(examId))
{
continue;
}
if (score.TryGetValue(examId, out var existingScore))
{
existingScore.Number = reader.GetValue(0)?.ToString();
existingScore.RemarkExamOrder = reader.GetValue(3)?.ToString() ?? string.Empty;
existingScore.LastUpdatedAt = DateTime.Now;
existingScore.LastUpdateUserId = job.UserId ?? "";
existingScore.LastUpdateFullName = job.FullName ?? "System Administrator";
batchUpdates.Add(existingScore);
batchCount++;
}
if (batchCount >= batchSize)
{
await _context.BulkUpdateAsync(batchUpdates);
batchUpdates.Clear();
batchCount = 0;
}
}
// Process remaining records
if (batchUpdates.Count > 0)
{
await _context.BulkUpdateAsync(batchUpdates);
}
} while (reader.NextResult());
}
#endregion
#region Helpers
private static int GetColumnIndex(string[] columns, string name, bool partial = false)
{
try
{
if (partial)
return Array.FindIndex(columns, x => x.Contains(name)) + 1;
else
return Array.FindIndex(columns, x => x == name) + 1;
}
catch
{
return 0;
}
}
/// <summary>
/// Get string value from ExcelDataReader by header column name
/// </summary>
private static string GetCellValue(IExcelDataReader reader, string[] cols, string headerName)
{
var idx = GetColumnIndex(cols, headerName);
if (idx <= 0 || idx > reader.FieldCount) return "";
return reader.GetValue(idx - 1)?.ToString() ?? "";
}
/// <summary>
/// Get DateTime value from ExcelDataReader by header column name
/// </summary>
private static DateTime? GetCellDateTime(IExcelDataReader reader, string[] cols, string headerName)
{
var idx = GetColumnIndex(cols, headerName);
if (idx <= 0 || idx > reader.FieldCount) return null;
var val = reader.GetValue(idx - 1);
if (val is DateTime dt) return dt;
if (val != null && DateTime.TryParse(val.ToString(), out var parsed)) return parsed;
return null;
}
/// <summary>
/// Get double value from ExcelDataReader by header column name
/// </summary>
private static double GetCellDouble(IExcelDataReader reader, string[] cols, string headerName)
{
var idx = GetColumnIndex(cols, headerName);
if (idx <= 0 || idx > reader.FieldCount) return 0.0;
var val = reader.GetValue(idx - 1);
if (val is double d) return d;
if (val != null && double.TryParse(val.ToString(), out var parsed)) return parsed;
return 0.0;
}
/// <summary>
/// Get decimal value from ExcelDataReader by header column name
/// </summary>
private static decimal GetCellDecimal(IExcelDataReader reader, string[] cols, string headerName)
{
var idx = GetColumnIndex(cols, headerName);
if (idx <= 0 || idx > reader.FieldCount) return 0m;
var val = reader.GetValue(idx - 1);
if (val is decimal dec) return dec;
if (val is double dbl) return (decimal)dbl;
if (val != null && decimal.TryParse(val.ToString(), out var parsed)) return parsed;
return 0m;
}
/// <summary>
/// Get double value from ExcelDataReader by 0-based column index
/// </summary>
private static double GetReaderDouble(IExcelDataReader reader, int index)
{
if (index < 0 || index >= reader.FieldCount) return 0.0;
var val = reader.GetValue(index);
if (val is double d) return d;
if (val != null && double.TryParse(val.ToString(), out var parsed)) return parsed;
return 0.0;
}
/// <summary>
/// Get decimal value from ExcelDataReader by 0-based column index
/// </summary>
private static decimal GetReaderDecimal(IExcelDataReader reader, int index)
{
if (index < 0 || index >= reader.FieldCount) return 0m;
var val = reader.GetValue(index);
if (val is decimal dec) return dec;
if (val is double dbl) return (decimal)dbl;
if (val != null && decimal.TryParse(val.ToString(), out var parsed)) return parsed;
return 0m;
}
#endregion
}
// Helper class to wrap a file path as IFormFile for MinIO upload
internal class DummyFormFile : IFormFile
{
private readonly string _filePath;
private readonly FileInfo _fileInfo;
public DummyFormFile(string filePath)
{
_filePath = filePath;
_fileInfo = new FileInfo(filePath);
}
public string ContentType => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
public string ContentDisposition => $"form-data; name=\"file\"; filename=\"{_fileInfo.Name}\"";
public IHeaderDictionary Headers => new HeaderDictionary();
public long Length => _fileInfo.Length;
public string Name => "file";
public string FileName => _fileInfo.Name;
public void CopyTo(Stream target)
{
using var stream = _fileInfo.OpenRead();
stream.CopyTo(target);
}
public async Task CopyToAsync(Stream target, CancellationToken cancellationToken = default)
{
using var stream = _fileInfo.OpenRead();
await stream.CopyToAsync(target, cancellationToken);
}
public Stream OpenReadStream()
{
return _fileInfo.OpenRead();
}
}