hrms-api-recruit/Services/ImportBackgroundService.cs

using System.Threading.Channels;
using BMA.EHR.Recruit.Core;
using BMA.EHR.Recruit.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using System.Net.Http.Headers;
using BMA.EHR.Recruit.Data;
using BMA.EHR.Recruit.Extensions;
using EFCore.BulkExtensions;
using BMA.EHR.Recruit.Models.Recruits;
using BMA.EHR.Recruit.Requests.Recruits;
using ExcelDataReader;

namespace BMA.EHR.Recruit.Services;

public class ImportBackgroundService : BackgroundService
{
    private readonly ImportJobQueue _queue;
    private readonly ImportJobTracker _tracker;
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly ILogger<ImportBackgroundService> _logger;

    public ImportBackgroundService(
        ImportJobQueue queue,
        ImportJobTracker tracker,
        IServiceScopeFactory scopeFactory,
        ILogger<ImportBackgroundService> logger)
    {
        _queue = queue;
        _tracker = tracker;
        _scopeFactory = scopeFactory;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("ImportBackgroundService started.");

        while (!stoppingToken.IsCancellationRequested)
        {
            var job = await _queue.DequeueAsync(stoppingToken);

            using var scope = _scopeFactory.CreateScope();
            var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
            var minioService = scope.ServiceProvider.GetRequiredService<MinIOService>();
            var recruitService = scope.ServiceProvider.GetRequiredService<RecruitService>();
            var notificationService = scope.ServiceProvider.GetRequiredService<NotificationService>();
            var webHostEnv = scope.ServiceProvider.GetRequiredService<IWebHostEnvironment>();
            var logger = scope.ServiceProvider.GetRequiredService<ILogger<ImportBackgroundService>>();

            try
            {
                _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running);

                switch (job.JobType)
                {
                    case ImportJobType.CandidateFile:
                        await ProcessCandidateFileAsync(context, minioService, webHostEnv, job);
                        break;
                    case ImportJobType.CandidateFileById:
                        await ProcessCandidateFileByIdAsync(context, minioService, recruitService, webHostEnv, job);
                        break;
                    case ImportJobType.ScoreFile:
                        await ProcessScoreFileAsync(context, minioService, recruitService, job);
                        break;
                    case ImportJobType.ResultFile:
                        await ProcessResultFileAsync(context, recruitService, job);
                        break;
                }

                _tracker.UpdateStatus(job.JobId, ImportJobStatus.Completed, job.TotalCount);

                await notificationService.SendImportNotificationAsync(job.Token, false, "ระบบนำเข้าข้อมูลสำเร็จ");
                job.Token = null; // Clear token after notification sent
            }
            catch (Exception ex)
            {
                logger.LogError(ex, "Import job {JobId} failed: {Message}", job.JobId, ex.Message);
                _tracker.UpdateStatus(job.JobId, ImportJobStatus.Failed, 0, ex.Message);

                try { await notificationService.SendImportNotificationAsync(job.Token, true, ex.Message); } catch { }
                job.Token = null; // Clear token after notification sent

                // cleanup minio file on failure
                if (!string.IsNullOrEmpty(job.ImportDocId))
                {
                    try { await minioService.DeleteFileAsync(Guid.Parse(job.ImportDocId)); } catch { }
                }
            }
            finally
            {
                // cleanup temp file
                try
                {
                    if (System.IO.File.Exists(job.ImportFile))
                        System.IO.File.Delete(job.ImportFile);
                }
                catch { }
            }
        }
    }

    #region CandidateFile

    private async Task ProcessCandidateFileAsync(ApplicationDbContext _context, MinIOService _minioService, IWebHostEnvironment _webHostEnv, ImportJobInfo job)
    {
        var imported = await _context.RecruitImports.FindAsync(job.RecruitImportId);
        if (imported == null) throw new Exception("RecruitImport not found");

        System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
        using var stream = System.IO.File.OpenRead(job.ImportFile);
        using var reader = ExcelReaderFactory.CreateReader(stream);

        do
        {
            // Read header row (row 1) to build column index map
            if (!reader.Read()) continue;
            var cols = new string[reader.FieldCount];
            for (int c = 0; c < reader.FieldCount; c++)
                cols[c] = reader.GetValue(c)?.ToString() ?? "";

            int batchCount = 0;
            const int batchSize = 500;
            int totalProcessed = 0;

            var batchRecruits = new List<Models.Recruits.Recruit>();
            var batchEducations = new List<RecruitEducation>();
            var batchOccupations = new List<RecruitOccupation>();
            var batchAddresses = new List<RecruitAddress>();
            var batchPayments = new List<RecruitPayment>();
            var batchCertificates = new List<RecruitCertificate>();

            while (reader.Read())
            {
                var cell1 = reader.GetValue(0)?.ToString();
                if (string.IsNullOrEmpty(cell1)) break;

                var r = new Models.Recruits.Recruit();
                r.ExamId = GetCellValue(reader, cols, CandidateFileHeader.ExamID);
                r.CitizenId = GetCellValue(reader, cols, CandidateFileHeader.PersonalID);
                r.Prefix = GetCellValue(reader, cols, CandidateFileHeader.Prefix);
                r.FirstName = GetCellValue(reader, cols, CandidateFileHeader.FirstName);
                r.LastName = GetCellValue(reader, cols, CandidateFileHeader.LastName);
                r.Gendor = GetCellValue(reader, cols, CandidateFileHeader.Gender);
                r.National = GetCellValue(reader, cols, CandidateFileHeader.National).IsNull("");
                r.Race = GetCellValue(reader, cols, CandidateFileHeader.Race).IsNull("");
                r.Religion = GetCellValue(reader, cols, CandidateFileHeader.Religion).IsNull("");
                r.DateOfBirth = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.DateOfBirth).ToDateTime(DateTimeFormat.Ymd, "-"));
                r.Marry = GetCellValue(reader, cols, CandidateFileHeader.Marry);
                r.Isspecial = "N";
                r.CitizenCardIssuer = GetCellValue(reader, cols, CandidateFileHeader.PersonalCardIssue);
                r.CitizenCardExpireDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.PersonalCardExpireDate).ToDateTime(DateTimeFormat.Ymd, "-"));
                r.ApplyDate = GetCellDateTime(reader, cols, CandidateFileHeader.ApplyDate) ?? DateTime.MinValue;
                r.PositionName = GetCellValue(reader, cols, CandidateFileHeader.PositionName).IsNull("");
                r.PositionType = GetCellValue(reader, cols, CandidateFileHeader.PositionType).IsNull("");
                r.PositionLevel = GetCellValue(reader, cols, CandidateFileHeader.PositionLevel).IsNull("");

                // address
                var address = new RecruitAddress()
                {
                    Address = GetCellValue(reader, cols, CandidateFileHeader.Address),
                    Moo = GetCellValue(reader, cols, CandidateFileHeader.Moo),
                    Soi = GetCellValue(reader, cols, CandidateFileHeader.Soi),
                    Road = GetCellValue(reader, cols, CandidateFileHeader.Road),
                    District = GetCellValue(reader, cols, CandidateFileHeader.District),
                    Amphur = GetCellValue(reader, cols, CandidateFileHeader.Amphur),
                    Province = GetCellValue(reader, cols, CandidateFileHeader.Province),
                    ZipCode = GetCellValue(reader, cols, CandidateFileHeader.ZipCode),
                    Telephone = GetCellValue(reader, cols, CandidateFileHeader.Telephone),
                    Mobile = GetCellValue(reader, cols, CandidateFileHeader.Mobile),
                    Address1 = GetCellValue(reader, cols, CandidateFileHeader.Address1),
                    Moo1 = GetCellValue(reader, cols, CandidateFileHeader.Moo1),
                    Soi1 = GetCellValue(reader, cols, CandidateFileHeader.Soi1),
                    Road1 = GetCellValue(reader, cols, CandidateFileHeader.Road1),
                    District1 = GetCellValue(reader, cols, CandidateFileHeader.District1),
                    Amphur1 = GetCellValue(reader, cols, CandidateFileHeader.Amphur1),
                    Province1 = GetCellValue(reader, cols, CandidateFileHeader.Province1),
                    ZipCode1 = GetCellValue(reader, cols, CandidateFileHeader.ZipCode1),
                };

                // payment
                var payment = new RecruitPayment()
                {
                    PaymentId = GetCellValue(reader, cols, CandidateFileHeader.PaymentID),
                    CompanyCode = GetCellValue(reader, cols, CandidateFileHeader.CompanyCode),
                    TextFile = GetCellValue(reader, cols, CandidateFileHeader.TextFile),
                    BankCode = GetCellValue(reader, cols, CandidateFileHeader.BankCode),
                    AccountNumber = GetCellValue(reader, cols, CandidateFileHeader.AccouontNumer),
                    TransDate = GetCellValue(reader, cols, CandidateFileHeader.TransDate),
                    TransTime = GetCellValue(reader, cols, CandidateFileHeader.TransTime),
                    CustomerName = GetCellValue(reader, cols, CandidateFileHeader.CustomerName),
                    RefNo1 = GetCellValue(reader, cols, CandidateFileHeader.RefNo1),
                    TermBranch = GetCellValue(reader, cols, CandidateFileHeader.TermBranch),
                    TellerId = GetCellValue(reader, cols, CandidateFileHeader.TellerID),
                    CreditDebit = GetCellValue(reader, cols, CandidateFileHeader.CreditDebit),
                    PaymentType = GetCellValue(reader, cols, CandidateFileHeader.Type),
                    ChequeNo = GetCellValue(reader, cols, CandidateFileHeader.ChequeNo),
                    Amount = GetCellDecimal(reader, cols, CandidateFileHeader.Amount),
                    ChqueBankCode = GetCellValue(reader, cols, CandidateFileHeader.ChqBankCode)
                };

                // occupation
                var occupation = new RecruitOccupation()
                {
                    Occupation = GetCellValue(reader, cols, CandidateFileHeader.Occupation),
                    Position = GetCellValue(reader, cols, CandidateFileHeader.Position),
                    Workplace = GetCellValue(reader, cols, CandidateFileHeader.Workplace),
                    Telephone = GetCellValue(reader, cols, CandidateFileHeader.WorkplaceTelephone),
                    WorkAge = GetCellValue(reader, cols, CandidateFileHeader.WorkAge),
                };

                // certificate
                var certificate = new RecruitCertificate()
                {
                    CertificateNo = GetCellValue(reader, cols, CandidateFileHeader.CertificateNo),
                    Description = GetCellValue(reader, cols, CandidateFileHeader.CertificateDesc),
                    IssueDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.CertificateIssueDate).ToDateTime(DateTimeFormat.Ymd, "-")),
                    ExpiredDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.CertificateExpireDate).ToDateTime(DateTimeFormat.Ymd, "-"))
                };

                var education = new RecruitEducation()
                {
                    Degree = GetCellValue(reader, cols, CandidateFileHeader.Degree),
                    Major = GetCellValue(reader, cols, CandidateFileHeader.Major),
                    MajorGroupId = GetCellValue(reader, cols, CandidateFileHeader.MajorGroupID),
                    MajorGroupName = GetCellValue(reader, cols, CandidateFileHeader.MajorGroupName),
                    University = GetCellValue(reader, cols, CandidateFileHeader.University),
                    GPA = GetCellDouble(reader, cols, CandidateFileHeader.GPA),
                    Specialist = GetCellValue(reader, cols, CandidateFileHeader.SpecialList),
                    HighDegree = GetCellValue(reader, cols, CandidateFileHeader.HighDegree),
                    BachelorDate = Convert.ToDateTime(GetCellValue(reader, cols, CandidateFileHeader.BachelorDate).ToDateTime(DateTimeFormat.Ymd, "-"))
                };

                r.Addresses.Add(address);
                r.Payments.Add(payment);
                r.Occupations.Add(occupation);
                r.Certificates.Add(certificate);
                r.Educations.Add(education);

                r.RecruitImport = imported;
                batchRecruits.Add(r);
                batchAddresses.Add(address);
                batchPayments.Add(payment);
                batchOccupations.Add(occupation);
                batchCertificates.Add(certificate);
                batchEducations.Add(education);

                batchCount++;
                totalProcessed++;

                if (batchCount >= batchSize)
                {
                    await _context.BulkInsertAsync(batchRecruits, options => { options.SetOutputIdentity = true; });

                    for (int j = 0; j < batchRecruits.Count; j++)
                    {
                        batchAddresses[j].Recruit = batchRecruits[j];
                        batchPayments[j].Recruit = batchRecruits[j];
                        batchOccupations[j].Recruit = batchRecruits[j];
                        batchCertificates[j].Recruit = batchRecruits[j];
                        batchEducations[j].Recruit = batchRecruits[j];
                    }

                    await _context.BulkInsertAsync(batchAddresses);
                    await _context.BulkInsertAsync(batchPayments);
                    await _context.BulkInsertAsync(batchOccupations);
                    await _context.BulkInsertAsync(batchCertificates);
                    await _context.BulkInsertAsync(batchEducations);

                    _context.ChangeTracker.Clear();

                    batchRecruits.Clear();
                    batchAddresses.Clear();
                    batchPayments.Clear();
                    batchOccupations.Clear();
                    batchCertificates.Clear();
                    batchEducations.Clear();
                    batchCount = 0;
                    _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
                }
            }

            // Process remaining records
            if (batchRecruits.Count > 0)
            {
                await _context.BulkInsertAsync(batchRecruits, options => { options.SetOutputIdentity = true; });

                for (int j = 0; j < batchRecruits.Count; j++)
                {
                    batchAddresses[j].Recruit = batchRecruits[j];
                    batchPayments[j].Recruit = batchRecruits[j];
                    batchOccupations[j].Recruit = batchRecruits[j];
                    batchCertificates[j].Recruit = batchRecruits[j];
                    batchEducations[j].Recruit = batchRecruits[j];
                }

                await _context.BulkInsertAsync(batchAddresses);
                await _context.BulkInsertAsync(batchPayments);
                await _context.BulkInsertAsync(batchOccupations);
                await _context.BulkInsertAsync(batchCertificates);
                await _context.BulkInsertAsync(batchEducations);

                _context.ChangeTracker.Clear();
            }
        } while (reader.NextResult());

        job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
    }

    #endregion

    #region CandidateFileById

    private async Task ProcessCandidateFileByIdAsync(ApplicationDbContext _context, MinIOService _minioService, RecruitService _recruitService, IWebHostEnvironment _webHostEnv, ImportJobInfo job)
    {
        var imported = await _context.RecruitImports.FindAsync(job.RecruitImportId);
        if (imported == null) throw new Exception("RecruitImport not found");

        // Save import history using regular SaveChanges (small operation)
        imported.ImportHostories.Add(new RecruitImportHistory
        {
            Description = "นำเข้าข้อมูลผู้สมัครสอบแข่งขัน",
            CreatedAt = DateTime.Now,
            CreatedUserId = job.UserId ?? "",
            CreatedFullName = job.FullName ?? "System Administrator",
            LastUpdatedAt = DateTime.Now,
            LastUpdateUserId = job.UserId ?? "",
            LastUpdateFullName = job.FullName ?? "System Administrator",
        });
        await _context.SaveChangesAsync();
        _context.ChangeTracker.Clear();

        var importId = imported.Id;
        var importRef = _context.Attach(new RecruitImport { Id = importId }).Entity;

        System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
        using var stream = System.IO.File.OpenRead(job.ImportFile);
        using var reader = ExcelReaderFactory.CreateReader(stream);

        do
        {
            // Skip header row
            if (!reader.Read()) continue;

            int row = 2;
            int batchCount = 0;
            const int batchSize = 500;
            int totalProcessed = 0;

            var batchRecruits = new List<Models.Recruits.Recruit>();
            var batchEducations = new List<RecruitEducation>();
            var batchOccupations = new List<RecruitOccupation>();
            var batchAddresses = new List<RecruitAddress>();
            var batchPayments = new List<RecruitPayment>();

            while (reader.Read())
            {
                var cell1 = reader.GetValue(0)?.ToString();
                if (string.IsNullOrEmpty(cell1)) break;

                try
                {
                    var r = new Models.Recruits.Recruit();
                    r.Id = Guid.NewGuid();
                    r.ExamId = reader.GetValue(0)?.ToString() ?? "";
                    r.PositionName = reader.GetValue(2)?.ToString() ?? "";
                    r.HddPosition = reader.GetValue(3)?.ToString() ?? "";
                    r.Prefix = reader.GetValue(4)?.ToString() == "อื่น ๆ" ? reader.GetValue(5)?.ToString() ?? "" : reader.GetValue(4)?.ToString() ?? "";
                    r.FirstName = reader.GetValue(6)?.ToString() ?? "";
                    r.LastName = reader.GetValue(7)?.ToString() ?? "";
                    r.Gendor = reader.GetValue(97)?.ToString() ?? "";
                    r.National = reader.GetValue(8)?.ToString() ?? "";
                    r.Race = "";
                    r.Religion = reader.GetValue(9)?.ToString() ?? "";
                    r.DateOfBirth = !string.IsNullOrWhiteSpace(reader.GetValue(10)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(10)?.ToString() ?? "", "dd/MM/yyyy") : null;
                    r.CitizenId = reader.GetValue(11)?.ToString() ?? "";
                    r.typeTest = reader.GetValue(12)?.ToString() ?? "";
                    r.Marry = "";
                    r.Isspecial = "N";
                    r.CitizenCardExpireDate = null;
                    r.ModifiedDate = null;
                    r.ApplyDate = !string.IsNullOrWhiteSpace(reader.GetValue(86)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(86)?.ToString() ?? "", "dd/MM/yyyy") : null;
                    r.PositionType = "";
                    r.PositionLevel = "";
                    r.CreatedAt = DateTime.Now;
                    r.CreatedUserId = job.UserId ?? "";
                    r.CreatedFullName = job.FullName ?? "System Administrator";
                    r.LastUpdatedAt = DateTime.Now;
                    r.LastUpdateUserId = job.UserId ?? "";
                    r.LastUpdateFullName = job.FullName ?? "System Administrator";
                    r.RecruitImport = importRef;

                    // Store child entities in separate lists for bulk insert
                    var education = new RecruitEducation()
                    {
                        Id = Guid.NewGuid(),
                        Degree = reader.GetValue(17)?.ToString() ?? "",
                        Major = reader.GetValue(18)?.ToString() == "อื่น ๆ" ? reader.GetValue(19)?.ToString() ?? "" : reader.GetValue(18)?.ToString() ?? "",
                        MajorGroupId = "",
                        MajorGroupName = "",
                        University = reader.GetValue(20)?.ToString() == "อื่น ๆ" ? reader.GetValue(21)?.ToString() ?? "" : reader.GetValue(20)?.ToString() ?? "",
                        GPA = GetReaderDouble(reader, 25),
                        Specialist = "",
                        HighDegree = reader.GetValue(26)?.ToString() ?? "",
                        BachelorDate = !string.IsNullOrWhiteSpace(reader.GetValue(24)?.ToString()) ? _recruitService.CheckDateTime(reader.GetValue(24)?.ToString() ?? "", "dd/MM/yyyy") : null,
                        Recruit = r,
                        CreatedAt = DateTime.Now,
                        CreatedUserId = job.UserId ?? "",
                        CreatedFullName = job.FullName ?? "System Administrator",
                        LastUpdatedAt = DateTime.Now,
                        LastUpdateUserId = job.UserId ?? "",
                        LastUpdateFullName = job.FullName ?? "System Administrator"
                    };

                    var occupation = new RecruitOccupation()
                    {
                        Id = Guid.NewGuid(),
                        Occupation = reader.GetValue(32)?.ToString() == "อื่น ๆ" ? reader.GetValue(33)?.ToString() ?? "" : reader.GetValue(32)?.ToString() ?? "",
                        Position = reader.GetValue(36)?.ToString() ?? "",
                        Workplace = $"{(reader.GetValue(35)?.ToString() ?? "")} {(reader.GetValue(34)?.ToString() ?? "")}",
                        Telephone = "",
                        WorkAge = "",
                        Recruit = r,
                        CreatedAt = DateTime.Now,
                        CreatedUserId = job.UserId ?? "",
                        CreatedFullName = job.FullName ?? "System Administrator",
                        LastUpdatedAt = DateTime.Now,
                        LastUpdateUserId = job.UserId ?? "",
                        LastUpdateFullName = job.FullName ?? "System Administrator"
                    };

                    var address = new RecruitAddress()
                    {
                        Id = Guid.NewGuid(),
                        Address = $"{(reader.GetValue(48)?.ToString() ?? "")} {(reader.GetValue(49)?.ToString() ?? "")}",
                        Moo = reader.GetValue(50)?.ToString() ?? "",
                        Soi = reader.GetValue(51)?.ToString() ?? "",
                        Road = reader.GetValue(52)?.ToString() ?? "",
                        District = reader.GetValue(53)?.ToString() ?? "",
                        Amphur = reader.GetValue(54)?.ToString() ?? "",
                        Province = reader.GetValue(55)?.ToString() ?? "",
                        ZipCode = (reader.GetValue(56)?.ToString() ?? "").Trim(),
                        Telephone = reader.GetValue(57)?.ToString() ?? "",
                        Mobile = "",
                        Address1 = $"{(reader.GetValue(60)?.ToString() ?? "")} {(reader.GetValue(61)?.ToString() ?? "")}",
                        Moo1 = reader.GetValue(62)?.ToString() ?? "",
                        Soi1 = reader.GetValue(63)?.ToString() ?? "",
                        Road1 = reader.GetValue(64)?.ToString() ?? "",
                        District1 = reader.GetValue(65)?.ToString() ?? "",
                        Amphur1 = reader.GetValue(66)?.ToString() ?? "",
                        Province1 = reader.GetValue(67)?.ToString() ?? "",
                        ZipCode1 = (reader.GetValue(68)?.ToString() ?? "").Trim(),
                        Recruit = r,
                        CreatedAt = DateTime.Now,
                        CreatedUserId = job.UserId ?? "",
                        CreatedFullName = job.FullName ?? "System Administrator",
                        LastUpdatedAt = DateTime.Now,
                        LastUpdateUserId = job.UserId ?? "",
                        LastUpdateFullName = job.FullName ?? "System Administrator"
                    };

                    var payment = new RecruitPayment()
                    {
                        Id = Guid.NewGuid(),
                        PaymentId = reader.GetValue(103)?.ToString() ?? "",
                        CompanyCode = reader.GetValue(104)?.ToString() ?? "",
                        TextFile = reader.GetValue(105)?.ToString() ?? "",
                        BankCode = reader.GetValue(106)?.ToString() ?? "",
                        AccountNumber = reader.GetValue(107)?.ToString() ?? "",
                        TransDate = reader.GetValue(108)?.ToString() ?? "",
                        TransTime = reader.GetValue(109)?.ToString() ?? "",
                        CustomerName = reader.GetValue(110)?.ToString() ?? "",
                        RefNo1 = reader.GetValue(111)?.ToString() ?? "",
                        TermBranch = reader.GetValue(112)?.ToString() ?? "",
                        TellerId = reader.GetValue(113)?.ToString() ?? "",
                        CreditDebit = reader.GetValue(114)?.ToString() ?? "",
                        PaymentType = reader.GetValue(115)?.ToString() ?? "",
                        ChequeNo = reader.GetValue(116)?.ToString() ?? "",
                        Amount = GetReaderDecimal(reader, 117),
                        ChqueBankCode = reader.GetValue(118)?.ToString() ?? "",
                        Recruit = r,
                        CreatedAt = DateTime.Now,
                        CreatedUserId = job.UserId ?? "",
                        CreatedFullName = job.FullName ?? "System Administrator",
                        LastUpdatedAt = DateTime.Now,
                        LastUpdateUserId = job.UserId ?? "",
                        LastUpdateFullName = job.FullName ?? "System Administrator"
                    };

                    batchRecruits.Add(r);
                    batchEducations.Add(education);
                    batchOccupations.Add(occupation);
                    batchAddresses.Add(address);
                    batchPayments.Add(payment);
                }
                catch (Exception ex)
                {
                    throw new Exception($"Row {row}: {ex.Message}", ex);
                }

                row++;
                batchCount++;
                totalProcessed++;

                if (batchCount >= batchSize)
                {
                    try
                    {
                        await _context.BulkInsertAsync(batchRecruits);
                        await _context.BulkInsertAsync(batchEducations);
                        await _context.BulkInsertAsync(batchOccupations);
                        await _context.BulkInsertAsync(batchAddresses);
                        await _context.BulkInsertAsync(batchPayments);
                    }
                    catch (Exception ex)
                    {
                        var batchStartRow = row - batchCount + 1;
                        throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
                    }

                    _context.ChangeTracker.Clear();

                    // Clear all lists for next batch
                    batchRecruits.Clear();
                    batchEducations.Clear();
                    batchOccupations.Clear();
                    batchAddresses.Clear();
                    batchPayments.Clear();
                    batchCount = 0;
                    _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
                }
            }

            // Process remaining records
            if (batchRecruits.Count > 0)
            {
                try
                {
                    await _context.BulkInsertAsync(batchRecruits);
                    await _context.BulkInsertAsync(batchEducations);
                    await _context.BulkInsertAsync(batchOccupations);
                    await _context.BulkInsertAsync(batchAddresses);
                    await _context.BulkInsertAsync(batchPayments);
                }
                catch (Exception ex)
                {
                    var batchStartRow = row - batchCount + 1;
                    throw new Exception($"BulkInsert failed (rows {batchStartRow}-{row - 1}, {batchRecruits.Count} records): {ex.InnerException?.Message ?? ex.Message}", ex);
                }

                _context.ChangeTracker.Clear();
            }
        } while (reader.NextResult());

        job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
    }

    #endregion

    #region ScoreFile

    private async Task ProcessScoreFileAsync(ApplicationDbContext _context, MinIOService _minioService, RecruitService _recruitService, ImportJobInfo job)
    {
        var rec_import = await _context.RecruitImports.AsQueryable()
                                .Include(x => x.ScoreImport)
                                .ThenInclude(x => x.Scores)
                                .Include(x => x.ImportHostories)
                                .FirstOrDefaultAsync(x => x.Id == job.RecruitImportId);

        if (rec_import == null) throw new Exception("RecruitImport not found");

        if (rec_import.ScoreImport != null && rec_import.ScoreImport.Scores != null)
        {
            await _context.BulkDeleteAsync(rec_import.ScoreImport.Scores.ToList());
        }

        rec_import.ImportHostories.Add(new RecruitImportHistory
        {
            Description = "นำเข้าข้อมูลผลคะแนนสอบ",
            CreatedAt = DateTime.Now,
            CreatedUserId = job.UserId ?? "",
            CreatedFullName = job.FullName ?? "System Administrator",
            LastUpdatedAt = DateTime.Now,
            LastUpdateUserId = job.UserId ?? "",
            LastUpdateFullName = job.FullName ?? "System Administrator",
        });

        // get doc from minio
        var doc = await _minioService.UploadFileAsync(new DummyFormFile(job.ImportFile));
        var imported = new ScoreImport
        {
            Year = rec_import.Year,
            ImportFile = doc,
            CreatedAt = DateTime.Now,
            CreatedUserId = job.UserId ?? "",
            CreatedFullName = job.FullName ?? "System Administrator",
            LastUpdatedAt = DateTime.Now,
            LastUpdateUserId = job.UserId ?? "",
            LastUpdateFullName = job.FullName ?? "System Administrator",
            Scores = new List<RecruitScore>()
        };

        // Save ScoreImport parent first to get its Id
        rec_import.ScoreImport = imported;
        await _context.SaveChangesAsync();
        _context.ChangeTracker.Clear();

        // preload recruits using AsNoTracking to avoid EF tracking overhead
        var recruitsDict = await _context.Recruits
            .AsNoTracking()
            .Where(x => x.RecruitImport.Id == rec_import.Id)
            .GroupBy(x => x.ExamId)
            .Where(g => g.Count() == 1)
            .Select(g => g.First())
            .ToDictionaryAsync(x => x.ExamId, x => x);

        System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
        using var stream = System.IO.File.OpenRead(job.ImportFile);
        using var reader = ExcelReaderFactory.CreateReader(stream);

        do
        {
            // Read header rows (rows 1-7), then data starts at row 8
            // Skip 7 rows: first 7 are header/metadata
            for (int skip = 0; skip < 7; skip++)
            {
                if (!reader.Read()) break;
            }

            var cols = new string[reader.FieldCount];
            // Use current row (row 7) as header reference — not actually used for ScoreFile column mapping
            // ScoreFile uses hardcoded column indices

            int batchCount = 0;
            const int batchSize = 500;
            int totalProcessed = 0;

            var batchScores = new List<RecruitScore>();

            while (reader.Read())
            {
                var cell1 = reader.GetValue(0)?.ToString();
                if (string.IsNullOrEmpty(cell1)) break;

                var r = new RecruitScore();
                r.ExamId = reader.GetValue(1)?.ToString();

                if (!string.IsNullOrEmpty(r.ExamId) && recruitsDict.TryGetValue(r.ExamId, out var recruit))
                {
                    r.CitizenId = reader.GetValue(2)?.ToString()?.Trim();
                    r.FullA = 200;
                    r.SumA = string.IsNullOrWhiteSpace(reader.GetValue(4)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(4)), 2);
                    r.PercentageA = string.IsNullOrWhiteSpace(reader.GetValue(5)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(5)), 2);
                    r.AStatus = string.IsNullOrWhiteSpace(reader.GetValue(6)?.ToString()) ? "" : reader.GetValue(6)?.ToString();
                    r.SumAB = string.IsNullOrWhiteSpace(reader.GetValue(4)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(4)), 2);
                    r.ABStatus = string.IsNullOrWhiteSpace(reader.GetValue(6)?.ToString()) ? "" : reader.GetValue(6)?.ToString();
                    r.FullC = 50;
                    r.SumC = string.IsNullOrWhiteSpace(reader.GetValue(7)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(7)), 2);
                    r.FullD = 50;
                    r.SumD = string.IsNullOrWhiteSpace(reader.GetValue(8)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(8)), 2);
                    r.SumCD = string.IsNullOrWhiteSpace(reader.GetValue(9)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(9)), 2);
                    r.PercentageC = string.IsNullOrWhiteSpace(reader.GetValue(10)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(10)), 2);
                    r.CStatus = string.IsNullOrWhiteSpace(reader.GetValue(11)?.ToString()) ? "" : reader.GetValue(11)?.ToString();
                    r.FullScore = 300;
                    r.TotalScore = string.IsNullOrWhiteSpace(reader.GetValue(12)?.ToString()) ? 0.00 : Math.Round(Convert.ToDouble(reader.GetValue(12)), 2);

                    var examStatusCol7 = reader.GetValue(6)?.ToString()?.Trim();
                    var examStatusCol14 = reader.GetValue(13)?.ToString()?.Trim();
                    r.ExamStatus =
                        examStatusCol7 == "ขาดสอบ" ? "ขส." :
                        examStatusCol14 == "ได้" ? "ผ่าน" :
                        examStatusCol14 == "ตก" ? "ไม่ผ่าน" : "-";

                    r.RemarkScore = string.IsNullOrWhiteSpace(reader.GetValue(14)?.ToString()) ? string.Empty : reader.GetValue(14)?.ToString();

                    var examAttr = reader.GetValue(15)?.ToString()?.Trim();
                    r.ExamAttribute =
                        examAttr == "ผ่าน" ? "มีคุณสมบัติ" :
                        examAttr == "ไม่ผ่าน" ? "ไม่มีคุณสมบัติ" : "";

                    r.Major = reader.Name; // worksheet name
                    r.CreatedAt = DateTime.Now;
                    r.CreatedUserId = job.UserId ?? "";
                    r.CreatedFullName = job.FullName ?? "System Administrator";
                    r.LastUpdatedAt = DateTime.Now;
                    r.LastUpdateUserId = job.UserId ?? "";
                    r.LastUpdateFullName = job.FullName ?? "System Administrator";
                    r.ScoreImport = imported;

                    batchScores.Add(r);
                }

                batchCount++;
                totalProcessed++;

                if (batchCount >= batchSize)
                {
                    await _context.BulkInsertAsync(batchScores);
                    batchScores.Clear();
                    batchCount = 0;
                    _tracker.UpdateStatus(job.JobId, ImportJobStatus.Running, totalProcessed);
                }
            }

            // Process remaining records
            if (batchScores.Count > 0)
            {
                await _context.BulkInsertAsync(batchScores);
            }
        } while (reader.NextResult());

        job.TotalCount = _tracker.GetJob(job.JobId)?.ProcessedCount ?? 0;
    }

    #endregion

    #region ResultFile

    private async Task ProcessResultFileAsync(ApplicationDbContext _context, RecruitService _recruitService, ImportJobInfo job)
    {
        var rec_import = await _context.RecruitImports.AsQueryable()
                                .Include(x => x.ScoreImport)
                                .ThenInclude(x => x.Scores)
                                .Include(x => x.ImportHostories)
                                .FirstOrDefaultAsync(x => x.Id == job.RecruitImportId);

        if (rec_import == null) throw new Exception("RecruitImport not found");

        // update old scores
        if (rec_import.ScoreImport != null && rec_import.ScoreImport.Scores != null)
        {
            var oldScores = rec_import.ScoreImport.Scores
                .Where(x => !string.IsNullOrEmpty(x.Number))
                .ToList();
            if (oldScores.Count > 0)
            {
                foreach (var x in oldScores)
                {
                    x.Number = string.Empty;
                    x.RemarkExamOrder = string.Empty;
                }
                await _context.BulkUpdateAsync(oldScores);
            }
        }

        rec_import.ImportHostories.Add(new RecruitImportHistory
        {
            Description = "นำเข้าข้อมูลผลการสอบ",
            CreatedAt = DateTime.Now,
            CreatedUserId = job.UserId ?? "",
            CreatedFullName = job.FullName ?? "System Administrator",
            LastUpdatedAt = DateTime.Now,
            LastUpdateUserId = job.UserId ?? "",
            LastUpdateFullName = job.FullName ?? "System Administrator",
        });
        await _context.SaveChangesAsync();
        _context.ChangeTracker.Clear();

        // preload scores using AsNoTracking to avoid EF tracking overhead
        var scoreList = await _context.RecruitScores
            .AsNoTracking()
            .Where(s => s.ScoreImport.RecruitImportId == rec_import.Id && !string.IsNullOrEmpty(s.ExamId))
            .GroupBy(x => x.ExamId)
            .Where(g => g.Count() == 1)
            .Select(g => g.First())
            .ToListAsync();
        var score = scoreList.ToDictionary(s => s.ExamId!, s => s);

        System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
        using var stream = System.IO.File.OpenRead(job.ImportFile);
        using var reader = ExcelReaderFactory.CreateReader(stream);

        do
        {
            // Skip 6 header rows, data starts at row 7
            for (int skip = 0; skip < 6; skip++)
            {
                if (!reader.Read()) break;
            }

            int batchCount = 0;
            const int batchSize = 500;
            var batchUpdates = new List<RecruitScore>();

            while (reader.Read())
            {
                var examId = reader.GetValue(1)?.ToString();
                if (string.IsNullOrWhiteSpace(examId))
                {
                    continue;
                }

                if (score.TryGetValue(examId, out var existingScore))
                {
                    existingScore.Number = reader.GetValue(0)?.ToString();
                    existingScore.RemarkExamOrder = reader.GetValue(3)?.ToString() ?? string.Empty;
                    existingScore.LastUpdatedAt = DateTime.Now;
                    existingScore.LastUpdateUserId = job.UserId ?? "";
                    existingScore.LastUpdateFullName = job.FullName ?? "System Administrator";
                    batchUpdates.Add(existingScore);
                    batchCount++;
                }

                if (batchCount >= batchSize)
                {
                    await _context.BulkUpdateAsync(batchUpdates);
                    batchUpdates.Clear();
                    batchCount = 0;
                }
            }

            // Process remaining records
            if (batchUpdates.Count > 0)
            {
                await _context.BulkUpdateAsync(batchUpdates);
            }
        } while (reader.NextResult());
    }

    #endregion

    #region Helpers

    private static int GetColumnIndex(string[] columns, string name, bool partial = false)
    {
        try
        {
            if (partial)
                return Array.FindIndex(columns, x => x.Contains(name)) + 1;
            else
                return Array.FindIndex(columns, x => x == name) + 1;
        }
        catch
        {
            return 0;
        }
    }

    /// <summary>
    /// Get string value from ExcelDataReader by header column name
    /// </summary>
    private static string GetCellValue(IExcelDataReader reader, string[] cols, string headerName)
    {
        var idx = GetColumnIndex(cols, headerName);
        if (idx <= 0 || idx > reader.FieldCount) return "";
        return reader.GetValue(idx - 1)?.ToString() ?? "";
    }

    /// <summary>
    /// Get DateTime value from ExcelDataReader by header column name
    /// </summary>
    private static DateTime? GetCellDateTime(IExcelDataReader reader, string[] cols, string headerName)
    {
        var idx = GetColumnIndex(cols, headerName);
        if (idx <= 0 || idx > reader.FieldCount) return null;
        var val = reader.GetValue(idx - 1);
        if (val is DateTime dt) return dt;
        if (val != null && DateTime.TryParse(val.ToString(), out var parsed)) return parsed;
        return null;
    }

    /// <summary>
    /// Get double value from ExcelDataReader by header column name
    /// </summary>
    private static double GetCellDouble(IExcelDataReader reader, string[] cols, string headerName)
    {
        var idx = GetColumnIndex(cols, headerName);
        if (idx <= 0 || idx > reader.FieldCount) return 0.0;
        var val = reader.GetValue(idx - 1);
        if (val is double d) return d;
        if (val != null && double.TryParse(val.ToString(), out var parsed)) return parsed;
        return 0.0;
    }

    /// <summary>
    /// Get decimal value from ExcelDataReader by header column name
    /// </summary>
    private static decimal GetCellDecimal(IExcelDataReader reader, string[] cols, string headerName)
    {
        var idx = GetColumnIndex(cols, headerName);
        if (idx <= 0 || idx > reader.FieldCount) return 0m;
        var val = reader.GetValue(idx - 1);
        if (val is decimal dec) return dec;
        if (val is double dbl) return (decimal)dbl;
        if (val != null && decimal.TryParse(val.ToString(), out var parsed)) return parsed;
        return 0m;
    }

    /// <summary>
    /// Get double value from ExcelDataReader by 0-based column index
    /// </summary>
    private static double GetReaderDouble(IExcelDataReader reader, int index)
    {
        if (index < 0 || index >= reader.FieldCount) return 0.0;
        var val = reader.GetValue(index);
        if (val is double d) return d;
        if (val != null && double.TryParse(val.ToString(), out var parsed)) return parsed;
        return 0.0;
    }

    /// <summary>
    /// Get decimal value from ExcelDataReader by 0-based column index
    /// </summary>
    private static decimal GetReaderDecimal(IExcelDataReader reader, int index)
    {
        if (index < 0 || index >= reader.FieldCount) return 0m;
        var val = reader.GetValue(index);
        if (val is decimal dec) return dec;
        if (val is double dbl) return (decimal)dbl;
        if (val != null && decimal.TryParse(val.ToString(), out var parsed)) return parsed;
        return 0m;
    }

    #endregion
}

// Helper class to wrap a file path as IFormFile for MinIO upload
internal class DummyFormFile : IFormFile
{
    private readonly string _filePath;
    private readonly FileInfo _fileInfo;

    public DummyFormFile(string filePath)
    {
        _filePath = filePath;
        _fileInfo = new FileInfo(filePath);
    }

    public string ContentType => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
    public string ContentDisposition => $"form-data; name=\"file\"; filename=\"{_fileInfo.Name}\"";
    public IHeaderDictionary Headers => new HeaderDictionary();
    public long Length => _fileInfo.Length;
    public string Name => "file";
    public string FileName => _fileInfo.Name;

    public void CopyTo(Stream target)
    {
        using var stream = _fileInfo.OpenRead();
        stream.CopyTo(target);
    }

    public async Task CopyToAsync(Stream target, CancellationToken cancellationToken = default)
    {
        using var stream = _fileInfo.OpenRead();
        await stream.CopyToAsync(target, cancellationToken);
    }

    public Stream OpenReadStream()
    {
        return _fileInfo.OpenRead();
    }
}